From b246d622ca05360b196c1eab9312f0251fc1d8cf Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 7 Sep 2023 08:04:55 -0700 Subject: [PATCH 01/93] add pipeline for api view copilot (#6915) --- packages/python-packages/apiview-gpt/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/python-packages/apiview-gpt/ci.yml diff --git a/packages/python-packages/apiview-gpt/ci.yml b/packages/python-packages/apiview-gpt/ci.yml new file mode 100644 index 00000000000..481369c16ff --- /dev/null +++ b/packages/python-packages/apiview-gpt/ci.yml @@ -0,0 +1,17 @@ +# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. +trigger: + branches: + include: + - main + paths: + include: + - packages/python-packages/apiview-gpt + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-tool-python.yml + parameters: + PythonVersion: '3.10' + PackagePath: 'packages/python-packages/apiview-gpt' + FeedName: 'public/azure-sdk-for-python' + ArtifactName: 'apiviewcopilot' + PackageName: 'Python API View Copilot' From c0d5dadd33760d40849f5008175706f22f8fab02 Mon Sep 17 00:00:00 2001 From: Konrad Jamrozik Date: Thu, 7 Sep 2023 14:21:43 -0700 Subject: [PATCH 02/93] Bugfixes to Verify-Links.ps1 (#6903) * fixes * ongoing * ongoing * add top-level throw/catches showing exception info * fix handling of cases when there is 1 link and when there is no RetryAfter.Delta * handle lack of Exception.Headers property * handle gracefully obtaining status code from $_.Exception.InnerException.ErrorCode --- eng/common/scripts/Verify-Links.ps1 | 160 ++++++++++++++++++---------- 1 file changed, 106 insertions(+), 54 deletions(-) diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 index 4bfc2124366..a77f57133f2 100644 --- a/eng/common/scripts/Verify-Links.ps1 +++ b/eng/common/scripts/Verify-Links.ps1 @@ -12,13 +12,14 @@ Specifies the file that contains a set of links to ignore when verifying. .PARAMETER devOpsLogging - Switch that will enable devops specific logging for warnings + Switch that will enable devops specific logging for warnings. .PARAMETER recursive - Check the links recurisvely based on recursivePattern. + Check the links recurisvely. Applies to links starting with 'baseUrl' parameter. Defaults to true. .PARAMETER baseUrl Recursively check links for all links verified that begin with this baseUrl, defaults to the folder the url is contained in. + If 'recursive' parameter is set to false, this parameter has no effect. .PARAMETER rootUrl Path to the root of the site for resolving rooted relative links, defaults to host root for http and file directory for local files. @@ -74,6 +75,8 @@ param ( [string] $requestTimeoutSec = 15 ) +Set-StrictMode -Version 3.0 + $ProgressPreference = "SilentlyContinue"; # Disable invoke-webrequest progress dialog # Regex of the locale keywords. $locale = "/en-us/" @@ -184,11 +187,15 @@ function ParseLinks([string]$baseUri, [string]$htmlContent) #$hrefs | Foreach-Object { Write-Host $_ } Write-Verbose "Found $($hrefs.Count) raw href's in page $baseUri"; - $links = $hrefs | ForEach-Object { ResolveUri $baseUri $_.Groups["href"].Value } + [string[]] $links = $hrefs | ForEach-Object { ResolveUri $baseUri $_.Groups["href"].Value } #$links | Foreach-Object { Write-Host $_ } - return $links + if ($null -eq $links) { + $links = @() + } + + return ,$links } function CheckLink ([System.Uri]$linkUri, $allowRetry=$true) @@ -239,11 +246,27 @@ function CheckLink ([System.Uri]$linkUri, $allowRetry=$true) } } catch { - $statusCode = $_.Exception.Response.StatusCode.value__ + + $responsePresent = $_.Exception.psobject.Properties.name -contains "Response" + if ($responsePresent) { + $statusCode = $_.Exception.Response.StatusCode.value__ + } else { + $statusCode = $null + } - if(!$statusCode) { + if (!$statusCode) { # Try to pull the error code from any inner SocketException we might hit - $statusCode = $_.Exception.InnerException.ErrorCode + + $innerExceptionPresent = $_.Exception.psobject.Properties.name -contains "InnerException" + + $errorCodePresent = $false + if ($innerExceptionPresent) { + $errorCodePresent = $_.Exception.InnerException.psobject.Properties.name -contains "ErrorCode" + } + + if ($errorCodePresent) { + $statusCode = $_.Exception.InnerException.ErrorCode + } } if ($statusCode -in $errorStatusCodes) { @@ -257,13 +280,30 @@ function CheckLink ([System.Uri]$linkUri, $allowRetry=$true) $linkValid = $false } else { + if ($null -ne $statusCode) { + # For 429 rate-limiting try to pause if possible - if ($allowRetry -and $_.Exception.Response -and $statusCode -eq 429) { - $retryAfter = $_.Exception.Response.Headers.RetryAfter.Delta.TotalSeconds + if ($allowRetry -and $responsePresent -and $statusCode -eq 429) { + + $headersPresent = $_.Exception.psobject.Properties.name -contains "Headers" + + $retryAfterPresent = $false + if ($headersPresent) { + $retryAfterPresent = $_.Exception.Headers.psobject.Properties.name -contains "RetryAfter" + } + + $retryAfterDeltaPresent = $false + if ($retryAfterPresent) { + $retryAfterDeltaPresent = $_.Exception.Headers.RetryAfter.psobject.Properties.name -contains "Delta" + } + + if ($retryAfterDeltaPresent) { + $retryAfter = $_.Exception.Response.Headers.RetryAfter.Delta.TotalSeconds + } # Default retry after 60 (arbitrary) seconds if no header given - if (!$retryAfter -or $retryAfter -gt 60) { $retryAfter = 60 } + if (!$retryAfterDeltaPresent -or $retryAfter -gt 60) { $retryAfter = 60 } Write-Host "Rate-Limited for $retryAfter seconds while requesting $linkUri" Start-Sleep -Seconds $retryAfter @@ -366,9 +406,9 @@ function GetLinks([System.Uri]$pageUri) LogError "Don't know how to process uri $pageUri" } - $links = ParseLinks $pageUri $content + [string[]] $links = ParseLinks $pageUri $content - return $links; + return ,$links; } if ($urls) { @@ -433,59 +473,71 @@ if ($devOpsLogging) { while ($pageUrisToCheck.Count -ne 0) { $pageUri = $pageUrisToCheck.Dequeue(); - if ($checkedPages.ContainsKey($pageUri)) { continue } - $checkedPages[$pageUri] = $true; - - $linkUris = GetLinks $pageUri - Write-Host "Checking $($linkUris.Count) links found on page $pageUri"; - $badLinksPerPage = @(); - foreach ($linkUri in $linkUris) { - $isLinkValid = CheckLink $linkUri - if (!$isLinkValid -and !$badLinksPerPage.Contains($linkUri)) { - if (!$linkUri.ToString().Trim()) { - $linkUri = $emptyLinkMessage + Write-Verbose "Processing pageUri $pageUri" + try { + if ($checkedPages.ContainsKey($pageUri)) { continue } + $checkedPages[$pageUri] = $true; + + [string[]] $linkUris = GetLinks $pageUri + Write-Host "Checking $($linkUris.Count) links found on page $pageUri"; + $badLinksPerPage = @(); + foreach ($linkUri in $linkUris) { + $isLinkValid = CheckLink $linkUri + if (!$isLinkValid -and !$badLinksPerPage.Contains($linkUri)) { + if (!$linkUri.ToString().Trim()) { + $linkUri = $emptyLinkMessage + } + $badLinksPerPage += $linkUri } - $badLinksPerPage += $linkUri - } - if ($recursive -and $isLinkValid) { - if ($linkUri.ToString().StartsWith($baseUrl) -and !$checkedPages.ContainsKey($linkUri)) { - $pageUrisToCheck.Enqueue($linkUri); + if ($recursive -and $isLinkValid) { + if ($linkUri.ToString().StartsWith($baseUrl) -and !$checkedPages.ContainsKey($linkUri)) { + $pageUrisToCheck.Enqueue($linkUri); + } } } + if ($badLinksPerPage.Count -gt 0) { + $badLinks[$pageUri] = $badLinksPerPage + } + } catch { + Write-Host "Exception encountered while processing pageUri $pageUri : $($_.Exception)" + throw } - if ($badLinksPerPage.Count -gt 0) { - $badLinks[$pageUri] = $badLinksPerPage - } -} -if ($devOpsLogging) { - Write-Host "##[endgroup]" } -if ($badLinks.Count -gt 0) { - Write-Host "Summary of broken links:" -} -foreach ($pageLink in $badLinks.Keys) { - Write-Host "'$pageLink' has $($badLinks[$pageLink].Count) broken link(s):" - foreach ($brokenLink in $badLinks[$pageLink]) { - Write-Host " $brokenLink" +try { + if ($devOpsLogging) { + Write-Host "##[endgroup]" } -} -$linksChecked = $checkedLinks.Count - $cachedLinksCount + if ($badLinks.Count -gt 0) { + Write-Host "Summary of broken links:" + } + foreach ($pageLink in $badLinks.Keys) { + Write-Host "'$pageLink' has $($badLinks[$pageLink].Count) broken link(s):" + foreach ($brokenLink in $badLinks[$pageLink]) { + Write-Host " $brokenLink" + } + } -if ($badLinks.Count -gt 0) { - Write-Host "Checked $linksChecked links with $($badLinks.Count) broken link(s) found." -} -else { - Write-Host "Checked $linksChecked links. No broken links found." -} + $linksChecked = $checkedLinks.Count - $cachedLinksCount -if ($outputCacheFile) -{ - $goodLinks = $checkedLinks.Keys.Where({ "True" -eq $checkedLinks[$_].ToString() }) | Sort-Object + if ($badLinks.Count -gt 0) { + Write-Host "Checked $linksChecked links with $($badLinks.Count) broken link(s) found." + } + else { + Write-Host "Checked $linksChecked links. No broken links found." + } + + if ($outputCacheFile) + { + $goodLinks = $checkedLinks.Keys.Where({ "True" -eq $checkedLinks[$_].ToString() }) | Sort-Object - Write-Host "Writing the list of validated links to $outputCacheFile" - $goodLinks | Set-Content $outputCacheFile + Write-Host "Writing the list of validated links to $outputCacheFile" + $goodLinks | Set-Content $outputCacheFile + } +} catch { + Write-Host "Exception encountered after all pageUris have been processed : $($_.Exception)" + throw } exit $badLinks.Count From 1d90c40a6dec21dbe82c58b43ea8673e0c89f265 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 7 Sep 2023 14:54:01 -0700 Subject: [PATCH 03/93] add endpoints for other languages (#6920) * add endpoints for other languages * added more languages --- packages/python-packages/apiview-gpt/app.py | 58 ++++++++++++++++++- .../apiview-gpt/src/__init__.py | 8 +++ .../python-packages/apiview-gpt/src/_c_api.py | 6 ++ .../apiview-gpt/src/_cpp_api.py | 6 ++ .../apiview-gpt/src/_go_api.py | 6 ++ .../apiview-gpt/src/_java_api.py | 6 ++ .../apiview-gpt/src/_js_api.py | 6 ++ .../apiview-gpt/src/_net_api.py | 6 ++ .../apiview-gpt/src/_swift_api.py | 6 ++ .../apiview-gpt/src/_typespec_api.py | 6 ++ 10 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 packages/python-packages/apiview-gpt/src/_c_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_cpp_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_go_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_java_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_js_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_net_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_swift_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_typespec_api.py diff --git a/packages/python-packages/apiview-gpt/app.py b/packages/python-packages/apiview-gpt/app.py index b4cf49a74cf..e67930b0ccd 100644 --- a/packages/python-packages/apiview-gpt/app.py +++ b/packages/python-packages/apiview-gpt/app.py @@ -1,5 +1,5 @@ from flask import Flask, request, jsonify -from src import review_python +from src import review_python, review_java, review_cpp, review_go, review_js, review_net, review_c, review_swift, review_typespec app = Flask(__name__) @@ -9,3 +9,59 @@ def python_api_reviewer(): content = data['content'] result = review_python(content) return jsonify(result) + +@app.route('/java', methods=['POST']) +def java_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_java(content) + return jsonify(result) + +@app.route('/js', methods=['POST']) +def js_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_js(content) + return jsonify(result) + +@app.route('/net', methods=['POST']) +def net_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_net(content) + return jsonify(result) + +@app.route('/cpp', methods=['POST']) +def cpp_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_cpp(content) + return jsonify(result) + +@app.route('/go', methods=['POST']) +def go_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_go(content) + return jsonify(result) + +@app.route('/c', methods=['POST']) +def c_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_c(content) + return jsonify(result) + +@app.route('/swift', methods=['POST']) +def swift_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_swift(content) + return jsonify(result) + +@app.route('/typespec', methods=['POST']) +def typespec_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_typespec(content) + return jsonify(result) diff --git a/packages/python-packages/apiview-gpt/src/__init__.py b/packages/python-packages/apiview-gpt/src/__init__.py index e11c1904ea5..caf95373657 100644 --- a/packages/python-packages/apiview-gpt/src/__init__.py +++ b/packages/python-packages/apiview-gpt/src/__init__.py @@ -4,6 +4,14 @@ from ._version import VERSION from ._gpt_reviewer import GptReviewer from ._python_api import review_python +from ._java_api import review_java +from ._js_api import review_js +from ._net_api import review_net +from ._cpp_api import review_cpp +from ._go_api import review_go +from ._c_api import review_c +from ._swift_api import review_swift +from ._typespec_api import review_typespec from ._vector_db import VectorDB from ._models import GuidelinesResult, Violation, VectorDocument, VectorSearchResult diff --git a/packages/python-packages/apiview-gpt/src/_c_api.py b/packages/python-packages/apiview-gpt/src/_c_api.py new file mode 100644 index 00000000000..5f84a12b2c1 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_c_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_c(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "c") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_cpp_api.py b/packages/python-packages/apiview-gpt/src/_cpp_api.py new file mode 100644 index 00000000000..9be7468f9d1 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_cpp_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_cpp(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "cpp") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_go_api.py b/packages/python-packages/apiview-gpt/src/_go_api.py new file mode 100644 index 00000000000..85aa1e6459b --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_go_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_go(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "go") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_java_api.py b/packages/python-packages/apiview-gpt/src/_java_api.py new file mode 100644 index 00000000000..04e0749a38c --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_java_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_java(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "java") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_js_api.py b/packages/python-packages/apiview-gpt/src/_js_api.py new file mode 100644 index 00000000000..cb7e88b9294 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_js_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_js(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "js") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_net_api.py b/packages/python-packages/apiview-gpt/src/_net_api.py new file mode 100644 index 00000000000..68a5108f40d --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_net_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_net(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "c#") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_swift_api.py b/packages/python-packages/apiview-gpt/src/_swift_api.py new file mode 100644 index 00000000000..aa2ecb156ee --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_swift_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_swift(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "swift") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_typespec_api.py b/packages/python-packages/apiview-gpt/src/_typespec_api.py new file mode 100644 index 00000000000..19630c4c301 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_typespec_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_typespec(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "typespec") + return result.json() From 9e3e1ddbb0b17be566f083df04277e6b0d6d6567 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 7 Sep 2023 15:07:36 -0700 Subject: [PATCH 04/93] Update lang names (#6922) * add endpoints for other languages * added more languages * update names * rename langs --- packages/python-packages/apiview-gpt/app.py | 56 ++++++++++++------- .../apiview-gpt/src/__init__.py | 13 +++-- .../apiview-gpt/src/_android_api.py | 6 ++ .../src/{_js_api.py => _clang_api.py} | 4 +- .../src/{_net_api.py => _dotnet_api.py} | 4 +- .../apiview-gpt/src/_golang_api.py | 6 ++ .../src/{_c_api.py => _ios_api.py} | 4 +- .../src/{_go_api.py => _rest_api.py} | 4 +- .../apiview-gpt/src/_swift_api.py | 6 -- .../apiview-gpt/src/_typescript_api.py | 6 ++ .../apiview-gpt/src/_typespec_api.py | 6 -- 11 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 packages/python-packages/apiview-gpt/src/_android_api.py rename packages/python-packages/apiview-gpt/src/{_js_api.py => _clang_api.py} (55%) rename packages/python-packages/apiview-gpt/src/{_net_api.py => _dotnet_api.py} (54%) create mode 100644 packages/python-packages/apiview-gpt/src/_golang_api.py rename packages/python-packages/apiview-gpt/src/{_c_api.py => _ios_api.py} (56%) rename packages/python-packages/apiview-gpt/src/{_go_api.py => _rest_api.py} (56%) delete mode 100644 packages/python-packages/apiview-gpt/src/_swift_api.py create mode 100644 packages/python-packages/apiview-gpt/src/_typescript_api.py delete mode 100644 packages/python-packages/apiview-gpt/src/_typespec_api.py diff --git a/packages/python-packages/apiview-gpt/app.py b/packages/python-packages/apiview-gpt/app.py index e67930b0ccd..6ff08f5451c 100644 --- a/packages/python-packages/apiview-gpt/app.py +++ b/packages/python-packages/apiview-gpt/app.py @@ -1,5 +1,16 @@ from flask import Flask, request, jsonify -from src import review_python, review_java, review_cpp, review_go, review_js, review_net, review_c, review_swift, review_typespec +from src import ( + review_python, + review_java, + review_cpp, + review_golang, + review_typescript, + review_dotnet, + review_clang, + review_ios, + review_rest, + review_android, +) app = Flask(__name__) @@ -17,18 +28,18 @@ def java_api_reviewer(): result = review_java(content) return jsonify(result) -@app.route('/js', methods=['POST']) -def js_api_reviewer(): +@app.route('/typescript', methods=['POST']) +def typescript_api_reviewer(): data = request.get_json() content = data['content'] - result = review_js(content) + result = review_typescript(content) return jsonify(result) -@app.route('/net', methods=['POST']) -def net_api_reviewer(): +@app.route('/dotnet', methods=['POST']) +def dotnet_api_reviewer(): data = request.get_json() content = data['content'] - result = review_net(content) + result = review_dotnet(content) return jsonify(result) @app.route('/cpp', methods=['POST']) @@ -38,30 +49,37 @@ def cpp_api_reviewer(): result = review_cpp(content) return jsonify(result) -@app.route('/go', methods=['POST']) -def go_api_reviewer(): +@app.route('/golang', methods=['POST']) +def golang_api_reviewer(): data = request.get_json() content = data['content'] - result = review_go(content) + result = review_golang(content) return jsonify(result) -@app.route('/c', methods=['POST']) -def c_api_reviewer(): +@app.route('/clang', methods=['POST']) +def clang_api_reviewer(): data = request.get_json() content = data['content'] - result = review_c(content) + result = review_clang(content) return jsonify(result) -@app.route('/swift', methods=['POST']) -def swift_api_reviewer(): +@app.route('/ios', methods=['POST']) +def ios_api_reviewer(): data = request.get_json() content = data['content'] - result = review_swift(content) + result = review_ios(content) return jsonify(result) -@app.route('/typespec', methods=['POST']) -def typespec_api_reviewer(): +@app.route('/rest', methods=['POST']) +def rest_api_reviewer(): data = request.get_json() content = data['content'] - result = review_typespec(content) + result = review_rest(content) + return jsonify(result) + +@app.route('/android', methods=['POST']) +def android_api_reviewer(): + data = request.get_json() + content = data['content'] + result = review_android(content) return jsonify(result) diff --git a/packages/python-packages/apiview-gpt/src/__init__.py b/packages/python-packages/apiview-gpt/src/__init__.py index caf95373657..312f8591f0f 100644 --- a/packages/python-packages/apiview-gpt/src/__init__.py +++ b/packages/python-packages/apiview-gpt/src/__init__.py @@ -5,13 +5,14 @@ from ._gpt_reviewer import GptReviewer from ._python_api import review_python from ._java_api import review_java -from ._js_api import review_js -from ._net_api import review_net +from ._typescript_api import review_typescript +from ._dotnet_api import review_dotnet from ._cpp_api import review_cpp -from ._go_api import review_go -from ._c_api import review_c -from ._swift_api import review_swift -from ._typespec_api import review_typespec +from ._golang_api import review_golang +from ._clang_api import review_clang +from ._ios_api import review_ios +from ._rest_api import review_rest +from ._android_api import review_android from ._vector_db import VectorDB from ._models import GuidelinesResult, Violation, VectorDocument, VectorSearchResult diff --git a/packages/python-packages/apiview-gpt/src/_android_api.py b/packages/python-packages/apiview-gpt/src/_android_api.py new file mode 100644 index 00000000000..49b5e64ac41 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_android_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_android(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "android") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_js_api.py b/packages/python-packages/apiview-gpt/src/_clang_api.py similarity index 55% rename from packages/python-packages/apiview-gpt/src/_js_api.py rename to packages/python-packages/apiview-gpt/src/_clang_api.py index cb7e88b9294..11652d4bdfc 100644 --- a/packages/python-packages/apiview-gpt/src/_js_api.py +++ b/packages/python-packages/apiview-gpt/src/_clang_api.py @@ -1,6 +1,6 @@ from ._gpt_reviewer import GptReviewer -def review_js(code): +def review_clang(code): reviewer = GptReviewer() - result = reviewer.get_response(code, "js") + result = reviewer.get_response(code, "clang") return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_net_api.py b/packages/python-packages/apiview-gpt/src/_dotnet_api.py similarity index 54% rename from packages/python-packages/apiview-gpt/src/_net_api.py rename to packages/python-packages/apiview-gpt/src/_dotnet_api.py index 68a5108f40d..0ef8b63fb89 100644 --- a/packages/python-packages/apiview-gpt/src/_net_api.py +++ b/packages/python-packages/apiview-gpt/src/_dotnet_api.py @@ -1,6 +1,6 @@ from ._gpt_reviewer import GptReviewer -def review_net(code): +def review_dotnet(code): reviewer = GptReviewer() - result = reviewer.get_response(code, "c#") + result = reviewer.get_response(code, "dotnet") return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_golang_api.py b/packages/python-packages/apiview-gpt/src/_golang_api.py new file mode 100644 index 00000000000..ee0bbeae233 --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_golang_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_golang(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "golang") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_c_api.py b/packages/python-packages/apiview-gpt/src/_ios_api.py similarity index 56% rename from packages/python-packages/apiview-gpt/src/_c_api.py rename to packages/python-packages/apiview-gpt/src/_ios_api.py index 5f84a12b2c1..2faf594725c 100644 --- a/packages/python-packages/apiview-gpt/src/_c_api.py +++ b/packages/python-packages/apiview-gpt/src/_ios_api.py @@ -1,6 +1,6 @@ from ._gpt_reviewer import GptReviewer -def review_c(code): +def review_ios(code): reviewer = GptReviewer() - result = reviewer.get_response(code, "c") + result = reviewer.get_response(code, "ios") return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_go_api.py b/packages/python-packages/apiview-gpt/src/_rest_api.py similarity index 56% rename from packages/python-packages/apiview-gpt/src/_go_api.py rename to packages/python-packages/apiview-gpt/src/_rest_api.py index 85aa1e6459b..caf7471a834 100644 --- a/packages/python-packages/apiview-gpt/src/_go_api.py +++ b/packages/python-packages/apiview-gpt/src/_rest_api.py @@ -1,6 +1,6 @@ from ._gpt_reviewer import GptReviewer -def review_go(code): +def review_rest(code): reviewer = GptReviewer() - result = reviewer.get_response(code, "go") + result = reviewer.get_response(code, "rest") return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_swift_api.py b/packages/python-packages/apiview-gpt/src/_swift_api.py deleted file mode 100644 index aa2ecb156ee..00000000000 --- a/packages/python-packages/apiview-gpt/src/_swift_api.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._gpt_reviewer import GptReviewer - -def review_swift(code): - reviewer = GptReviewer() - result = reviewer.get_response(code, "swift") - return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_typescript_api.py b/packages/python-packages/apiview-gpt/src/_typescript_api.py new file mode 100644 index 00000000000..8f6941f25cb --- /dev/null +++ b/packages/python-packages/apiview-gpt/src/_typescript_api.py @@ -0,0 +1,6 @@ +from ._gpt_reviewer import GptReviewer + +def review_typescript(code): + reviewer = GptReviewer() + result = reviewer.get_response(code, "typescript") + return result.json() diff --git a/packages/python-packages/apiview-gpt/src/_typespec_api.py b/packages/python-packages/apiview-gpt/src/_typespec_api.py deleted file mode 100644 index 19630c4c301..00000000000 --- a/packages/python-packages/apiview-gpt/src/_typespec_api.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._gpt_reviewer import GptReviewer - -def review_typespec(code): - reviewer = GptReviewer() - result = reviewer.get_response(code, "typespec") - return result.json() From 6d9a10ab1ad9c4b7e11290dde803b123b2359325 Mon Sep 17 00:00:00 2001 From: Konrad Jamrozik Date: Fri, 8 Sep 2023 14:37:27 -0700 Subject: [PATCH 05/93] add SpecsPipelinesBuildsJobLogs (#6927) --- .../kojamroz/SpecsPipelinesBuildsJobLogs.kql | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql b/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql new file mode 100644 index 00000000000..a573450cabf --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql @@ -0,0 +1,65 @@ +.create-or-alter function with ( + folder='users/kojamroz', + docstring="Build logs for specific ADO job in the specs pipelines builds. See source for details." + ) + SpecsPipelinesBuildsJobLogs( + jobName:string, + start:datetime, + end:datetime, + buildId:string = "*") +{ +// ABOUT THIS FUNCTION +// +// This function returns build logs for the +// Azure.azure-rest-api-specs-pipeline [1] +// and Azure.azure-rest-api-specs-pipeline-staging [2] builds, +// for job named 'jobName', e.g. "BreakingChange" or "LintDiff". +// +// The logs are from time period between 'start' and 'end'. +// +// Optionally, you can pass 'buildId'. By default it is any build, denoted as "*". +// +// If you pass specific 'buildId', consider narrowing down 'start' and 'end' +// to the time period the build generated logs, to keep the function call performant. +// +// Run this function against cluster Azsdkengsys, database Pipelines. +// +// [1] 1736: Azure.azure-rest-api-specs-pipeline +// https://dev.azure.com/azure-sdk/internal/_build?definitionId=1736 +// +// [2] 3268: Azure.azure-rest-api-specs-pipeline-staging +// https://dev.azure.com/azure-sdk/internal/_build?definitionId=3268 +// +// --------------------------------------------------------------- +// +let filteredJobFirstMessage = strcat("##[section]Starting: ", jobName); +let allPipelineLogs = +cluster('azsdkengsys.westus2.kusto.windows.net').database('Pipelines').BuildLogLine + | where Timestamp between (start..end) + | where BuildDefinitionId in ("1736", "3268") + | where buildId == "*" or BuildId == buildId +; +let logIds = +allPipelineLogs + | where Message == filteredJobFirstMessage + | distinct BuildId, LogId + // Here we ensure only one LogId is taken from each BuildId. This is necessary as given LogId can appear twice: + // once with log line offset for given job, and once with offset for given job. + // Alternatively, the rows could be deduplicated by appropriate join to the BuildTimelineRecord table. + | summarize arg_min(LogId, *) by BuildId +; +let filteredLogs = +logIds + | join kind=inner allPipelineLogs on BuildId, LogId + | project-away BuildId1, LogId1 + | extend + PacificTime = datetime_utc_to_local(Timestamp, 'US/Pacific'), + buildUrl = strcat("https://dev.azure.com/azure-sdk/internal/_build/results?buildId=", BuildId, "&view=results") + | project-reorder Timestamp, PacificTime, buildUrl, BuildDefinitionId, BuildId, LineNumber, Message + // Get rid of duplicate logs. This is a known issue with no known proper fix - see the comment in the + // definition of "logIds" above. + | distinct * + | order by Timestamp desc +; +filteredLogs +} From ae455ff09661d41dc911bb0d17edc8563a3fab95 Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Sun, 10 Sep 2023 13:44:54 -0400 Subject: [PATCH 06/93] Create Epic work item type as Service or product (#6782) * Create Epic work item type as Service or product when running prepare release script --- .../Helpers/DevOps-WorkItem-Helpers.ps1 | 120 +++++++++++++++--- 1 file changed, 101 insertions(+), 19 deletions(-) diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 index fc0d3497c06..ef730282e24 100644 --- a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 +++ b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 @@ -135,7 +135,7 @@ function BuildHashKey() } $parentWorkItems = @{} -function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand = $false) +function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand = $false, $ignoreReleasePlannerTests = $true) { $key = BuildHashKey $serviceName $packageDisplayName if ($key -and $parentWorkItems.ContainsKey($key)) { @@ -154,10 +154,12 @@ function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand = else { $serviceCondition = "[ServiceName] <> ''" } - + if($ignoreReleasePlannerTests){ + $serviceCondition += " AND [Tags] NOT CONTAINS 'Release Planner App Test'" + } $query = "SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Epic' AND ${serviceCondition}" - $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent") + $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent", "System.Tags") $workItems = Invoke-Query $fields $query $outputCommand @@ -180,13 +182,63 @@ function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand = return $null } +$releasePlanWorkItems = @{} +function FindReleasePlanWorkItem($serviceName, $packageDisplayName, $outputCommand = $false, $ignoreReleasePlannerTests = $true) +{ + $key = BuildHashKey $serviceName $packageDisplayName + if ($key -and $releasePlanWorkItems.ContainsKey($key)) { + return $releasePlanWorkItems[$key] + } + + if ($serviceName) { + $condition = "[ServiceName] = '${serviceName}'" + if ($packageDisplayName) { + $condition += " AND [PackageDisplayName] = '${packageDisplayName}'" + } + else { + $condition += " AND [PackageDisplayName] = ''" + } + } + else { + $condition = "[ServiceName] <> ''" + } + $condition += " AND [System.State] <> 'Finished'" + if($ignoreReleasePlannerTests){ + $condition += " AND [Tags] NOT CONTAINS 'Release Planner App Test'" + } + + $query = "SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Release Plan' AND ${condition}" + + $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent", "System.Tags") + + $workItems = Invoke-Query $fields $query $outputCommand + + foreach ($wi in $workItems) + { + $localKey = BuildHashKey $wi.fields["Custom.ServiceName"] $wi.fields["Custom.PackageDisplayName"] + if (!$localKey) { continue } + if ($releasePlanWorkItems.ContainsKey($localKey) -and $releasePlanWorkItems[$localKey].id -ne $wi.id) { + Write-Warning "Already found parent [$($releasePlanWorkItems[$localKey].id)] with key [$localKey], using that one instead of [$($wi.id)]." + } + else { + Write-Verbose "[$($wi.id)]$localKey - Cached" + $releasePlanWorkItems[$localKey] = $wi + } + } + + if ($key -and $releasePlanWorkItems.ContainsKey($key)) { + return $releasePlanWorkItems[$key] + } + return $null +} + $packageWorkItems = @{} $packageWorkItemWithoutKeyFields = @{} -function FindLatestPackageWorkItem($lang, $packageName, $outputCommand = $true) +function FindLatestPackageWorkItem($lang, $packageName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) { # Cache all the versions of this package and language work items - $null = FindPackageWorkItem $lang $packageName -includeClosed $true -outputCommand $outputCommand + $null = FindPackageWorkItem $lang $packageName -includeClosed $true -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests $latestWI = $null foreach ($wi in $packageWorkItems.Values) @@ -206,7 +258,7 @@ function FindLatestPackageWorkItem($lang, $packageName, $outputCommand = $true) return $latestWI } -function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $true, $includeClosed = $false) +function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $true, $includeClosed = $false, $ignoreReleasePlannerTests = $true) { $key = BuildHashKeyNoNull $lang $packageName $version if ($key -and $packageWorkItems.ContainsKey($key)) { @@ -218,6 +270,7 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr $fields += "System.State" $fields += "System.AssignedTo" $fields += "System.Parent" + $fields += "System.Tags" $fields += "Custom.Language" $fields += "Custom.Package" $fields += "Custom.PackageDisplayName" @@ -251,7 +304,9 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr if ($version) { $query += " AND [PackageVersionMajorMinor] = '${version}'" } - + if($ignoreReleasePlannerTests){ + $query += " AND [Tags] NOT CONTAINS 'Release Planner App Test'" + } $workItems = Invoke-Query $fields $query $outputCommand foreach ($wi in $workItems) @@ -277,13 +332,13 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr return $null } -function InitializeWorkItemCache($outputCommand = $true, $includeClosed = $false) +function InitializeWorkItemCache($outputCommand = $true, $includeClosed = $false, $ignoreReleasePlannerTests = $true) { # Pass null to cache all service parents - $null = FindParentWorkItem -serviceName $null -packageDisplayName $null -outputCommand $outputCommand + $null = FindParentWorkItem -serviceName $null -packageDisplayName $null -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests # Pass null to cache all the package items - $null = FindPackageWorkItem -lang $null -packageName $null -version $null -outputCommand $outputCommand -includeClosed $includeClosed + $null = FindPackageWorkItem -lang $null -packageName $null -version $null -outputCommand $outputCommand -includeClosed $includeClosed -ignoreReleasePlannerTests $ignoreReleasePlannerTests } function GetCachedPackageWorkItems() @@ -490,22 +545,46 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte } } - $newparentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false + $newparentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false UpdateWorkItemParent $existingItem $newParentItem -outputCommand $outputCommand return $existingItem } - $parentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false + $parentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false $workItem = CreateWorkItem $title "Package" "Release" "Release" $fields $assignedTo $parentItem.id -outputCommand $outputCommand Write-Host "[$($workItem.id)]$lang - $pkgName($verMajorMinor) - Created" return $workItem } -function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outputCommand = $true) +function FindOrCreateReleasePlanParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) +{ + $existingItem = FindReleasePlanWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests + if ($existingItem) { + Write-Host "Found existing release plan work item [$($existingItem.id)]" + $newparentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests + UpdateWorkItemParent $existingItem $newParentItem + return $existingItem + } + + $fields = @() + $fields += "`"PackageDisplayName=${packageDisplayName}`"" + $fields += "`"ServiceName=${serviceName}`"" + $productParentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests + $title = "Release Plan - $($packageDisplayName)" + $workItem = CreateWorkItem $title "Release Plan" "Release" "Release" $fields $null $productParentItem.id + + $localKey = BuildHashKey $serviceName $packageDisplayName + Write-Host "[$($workItem.id)]$localKey - Created release plan work item" + $releasePlanWorkItems[$localKey] = $workItem + return $workItem +} + +function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) { - $existingItem = FindParentWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand + $existingItem = FindParentWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests if ($existingItem) { - $newparentItem = FindOrCreateServiceParent $serviceName -outputCommand $outputCommand + Write-Host "Found existing product work item [$($existingItem.id)]" + $newparentItem = FindOrCreateServiceParent $serviceName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests UpdateWorkItemParent $existingItem $newParentItem return $existingItem } @@ -513,7 +592,8 @@ function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outp $fields = @() $fields += "`"PackageDisplayName=${packageDisplayName}`"" $fields += "`"ServiceName=${serviceName}`"" - $serviceParentItem = FindOrCreateServiceParent $serviceName -outputCommand $outputCommand + $fields += "`"Custom.EpicType=Product`"" + $serviceParentItem = FindOrCreateServiceParent $serviceName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests $workItem = CreateWorkItem $packageDisplayName "Epic" "Release" "Release" $fields $null $serviceParentItem.id $localKey = BuildHashKey $serviceName $packageDisplayName @@ -522,21 +602,23 @@ function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outp return $workItem } -function FindOrCreateServiceParent($serviceName, $outputCommand = $true) +function FindOrCreateServiceParent($serviceName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) { - $serviceParent = FindParentWorkItem $serviceName -outputCommand $outputCommand + $serviceParent = FindParentWorkItem $serviceName -packageDisplayName $null -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests if ($serviceParent) { + Write-Host "Found existing service work item [$($serviceParent.id)]" return $serviceParent } $fields = @() $fields += "`"PackageDisplayName=`"" $fields += "`"ServiceName=${serviceName}`"" + $fields += "`"Custom.EpicType=Service`"" $parentId = $null $workItem = CreateWorkItem $serviceName "Epic" "Release" "Release" $fields $null $parentId -outputCommand $outputCommand $localKey = BuildHashKey $serviceName - Write-Host "[$($workItem.id)]$localKey - Created" + Write-Host "[$($workItem.id)]$localKey - Created service work item" $parentWorkItems[$localKey] = $workItem return $workItem } From a1a90afcc06abfff9265782c4633c9fc3d7c0a64 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Sep 2023 10:57:53 -0700 Subject: [PATCH 07/93] Fixed several ApiView parser issues. (#6931) * Fixed issue creating ApiView for azure core; destructors that aren't virtual cause errors; overrides without the override keyword also cause errors; added output for override and final attributes * Corrected comment --- .../ApiViewProcessor/ApiViewMessage.cpp | 20 ++- .../ApiViewProcessor/ApiViewMessage.hpp | 2 + .../ApiViewProcessor/AstNode.cpp | 137 ++++++++++++++--- .../ApiViewProcessor/ProcessorImpl.cpp | 34 +++-- .../cpp-api-parser/ParseTests/CMakeLists.txt | 4 +- .../ParseTests/TestCases/DestructorTests.cpp | 140 ++++++++++++++++++ .../cpp-api-parser/ParseTests/tests.cpp | 42 +++++- 7 files changed, 346 insertions(+), 33 deletions(-) create mode 100644 tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DestructorTests.cpp diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp index 098d117209d..e47ac7586b4 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp @@ -67,14 +67,32 @@ void AzureClassesDatabase::CreateApiViewMessage( newMessage.Level = ApiViewMessage::MessageLevel::Info; break; } - case ApiViewMessages::UsingDirectiveFound: { + case ApiViewMessages::ImplicitOverride: { newMessage.DiagnosticId = "CPA0009"; + newMessage.DiagnosticText = "Implicit override of virtual method. Consider using the " + "'override' keyword to make the override semantics explicit."; + newMessage.Level = ApiViewMessage::MessageLevel::Info; + newMessage.HelpLinkUri = "https://isocpp.github.io/CppCoreGuidelines/" + "CppCoreGuidelines#c128-virtual-functions-should-specify-exactly-" + "one-of-virtual-override-or-final"; + break; + } + case ApiViewMessages::UsingDirectiveFound: { + newMessage.DiagnosticId = "CPA000A"; newMessage.DiagnosticText = "Using Namespace directive found in header file. "; newMessage.HelpLinkUri = "https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rs-using-directive"; newMessage.Level = ApiViewMessage::MessageLevel::Error; break; } + case ApiViewMessages::NonVirtualDestructor: { + newMessage.DiagnosticId = "CPA000B"; + newMessage.DiagnosticText = "Base class destructors should be public and virtual or protected and non-virtual. "; + newMessage.HelpLinkUri + = "https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#c35-a-base-class-destructor-should-be-either-public-and-virtual-or-protected-and-non-virtual"; + newMessage.Level = ApiViewMessage::MessageLevel::Error; + break; + } } newMessage.TargetId = targetId; m_diagnostics.push_back(std::move(newMessage)); diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp index 6026cd7a9b7..1c3e54d573c 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp @@ -34,4 +34,6 @@ enum class ApiViewMessages InternalTypesInNonCorePackage, // Internal types in a non-core package ImplicitConstructor, // Constructor for a type is not marked "explicit". UsingDirectiveFound, // "using namespace" directive found. + ImplicitOverride, // Implicit override of virtual method. + NonVirtualDestructor, // Destructor of non-final class is not virtual. }; diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp index f8efa485969..89d24e95027 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp @@ -573,8 +573,8 @@ class AstDeclRefExpr : public AstExpr { public: AstDeclRefExpr(DeclRefExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_referencedName{ - expression->getFoundDecl()->getQualifiedNameAsString()} + : AstExpr(expression, context), + m_referencedName{expression->getFoundDecl()->getQualifiedNameAsString()} { } virtual void Dump(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override @@ -1106,9 +1106,9 @@ AstNamedNode::AstNamedNode( NamedDecl const* namedDecl, AzureClassesDatabase* const database, std::shared_ptr parentNode) - : AstNode(namedDecl), - m_namespace{AstNode::GetNamespaceForDecl(namedDecl)}, m_name{namedDecl->getNameAsString()}, - m_classDatabase(database), m_navigationId{namedDecl->getQualifiedNameAsString()}, + : AstNode(namedDecl), m_namespace{AstNode::GetNamespaceForDecl(namedDecl)}, + m_name{namedDecl->getNameAsString()}, m_classDatabase(database), + m_navigationId{namedDecl->getQualifiedNameAsString()}, m_nodeDocumentation{AstNode::GetCommentForNode(namedDecl->getASTContext(), namedDecl)}, m_nodeAccess{namedDecl->getAccess()} { @@ -1472,8 +1472,8 @@ class AstTemplateTemplateParameter : public AstNamedNode { AzureClassesDatabase* const database, std::shared_ptr parentNode) : AstNamedNode(templateParam, database, parentNode), - m_paramName{templateParam->getNameAsString()}, m_isParameterPack{ - templateParam->isParameterPack()} + m_paramName{templateParam->getNameAsString()}, + m_isParameterPack{templateParam->isParameterPack()} { for (auto attr : templateParam->attrs()) { @@ -1487,7 +1487,7 @@ class AstTemplateTemplateParameter : public AstNamedNode { if (templateParam->hasDefaultArgument()) { - auto defaultArg = templateParam->getDefaultArgument().getArgument(); + auto const& defaultArg = templateParam->getDefaultArgument().getArgument(); switch (defaultArg.getKind()) { @@ -1813,9 +1813,13 @@ class AstFunction : public AstNamedNode { class AstMethod : public AstFunction { protected: - bool m_isVirtual; - bool m_isConst; - bool m_isPure; + bool m_isVirtual{}; + bool m_isConst{}; + bool m_isPure{}; + bool m_isOverride{}; + bool m_isImplicitOverride{}; + bool m_isFinal{}; + bool m_isImplicitFinal{}; RefQualifierKind m_refQualifier; public: @@ -1826,6 +1830,60 @@ class AstMethod : public AstFunction { : AstFunction(method, database, parentNode), m_isVirtual(method->isVirtual()), m_isPure(method->isPure()), m_isConst(method->isConst()) { + // We assume that this is an implicit override if there are overriden methods. If we later find + // an override attribute, we know it's not an implicit override. + // + // Note that we don't do this for destructors, because they typically won't have an override + // attribute. + // + // Also note that if size_overriden_methods is non-zero, it means that the base class method is + // already virtual. + // + if (method->getKind() == Decl::Kind::CXXMethod) + { + if (method->size_overridden_methods() > 0) + { + m_isOverride = true; + m_isImplicitOverride = true; + } + } + + for (auto& attr : method->attrs()) + { + auto location{attr->getLocation()}; + switch (attr->getKind()) + { + case attr::Override: + m_isImplicitOverride = false; + break; + case attr::Final: + if (attr->isImplicit()) + { + method->dump(llvm::outs()); + // database->CreateApiViewMessage(ApiViewMessages::ImplicitOverride, + // m_navigationId); + + m_isImplicitFinal = true; + } + else + { + m_isFinal = true; + } + break; + case attr::Deprecated: + break; + default: + llvm::outs() << "Unknown Method Attribute: "; + attr->printPretty(llvm::outs(), LangOptions()); + llvm::outs() << "\n"; + break; + } + } + if (m_isOverride && m_isImplicitOverride) + { + database->CreateApiViewMessage(ApiViewMessages::ImplicitOverride, m_navigationId); + } + auto typePtr = method->getType().getTypePtr()->castAs(); m_refQualifier = typePtr->getRefQualifier(); m_parentClass = method->getParent()->getNameAsString(); @@ -1876,6 +1934,16 @@ class AstMethod : public AstFunction { dumper->InsertWhitespace(); dumper->InsertLiteral("0"); } + if (m_isOverride) + { + dumper->InsertWhitespace(); + dumper->InsertKeyword("override"); + } + if (m_isFinal) + { + dumper->InsertWhitespace(); + dumper->InsertKeyword("final"); + } if (dumpOptions.NeedsTrailingSemi) { dumper->InsertPunctuation(';'); @@ -1963,6 +2031,7 @@ class AstDestructor : public AstMethod { bool m_isDefault{false}; bool m_isDeleted{false}; bool m_isExplicitlyDefaulted{false}; + bool m_isVirtual{false}; public: AstDestructor( @@ -1970,8 +2039,37 @@ class AstDestructor : public AstMethod { AzureClassesDatabase* const database, std::shared_ptr parentNode) : AstMethod(dtor, database, parentNode), m_isDefault{dtor->isDefaulted()}, - m_isDeleted{dtor->isDeleted()}, m_isExplicitlyDefaulted{dtor->isExplicitlyDefaulted()} + m_isDeleted{dtor->isDeleted()}, m_isExplicitlyDefaulted{dtor->isExplicitlyDefaulted()}, + m_isVirtual{dtor->isVirtual()} { + bool isBaseClassDtor{true}; + // If this destructor overrides a base class destructor, it's not a base class destructor. + if (dtor->size_overridden_methods() != 0) + { + isBaseClassDtor = false; + } + if (dtor->getAccess() == AS_protected) + { + if (dtor->isVirtual()) + { + database->CreateApiViewMessage(ApiViewMessages::NonVirtualDestructor, m_navigationId); + } + } + else if (dtor->getAccess() == AS_public) + { + if (!dtor->isVirtual()) + { + auto parentClass = dtor->getParent(); + if (!parentClass->isEffectivelyFinal()) + { + database->CreateApiViewMessage(ApiViewMessages::NonVirtualDestructor, m_navigationId); + } + } + } + else + { + database->CreateApiViewMessage(ApiViewMessages::NonVirtualDestructor, m_navigationId); + } } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { @@ -2380,9 +2478,9 @@ class AstField : public AstNamedNode { AzureClassesDatabase* const azureClassesDatabase, std::shared_ptr parentNode) : AstNamedNode(fieldDecl, azureClassesDatabase, parentNode), - m_fieldType{fieldDecl->getType()}, m_initializer{AstExpr::Create( - fieldDecl->getInClassInitializer(), - fieldDecl->getASTContext())}, + m_fieldType{fieldDecl->getType()}, + m_initializer{ + AstExpr::Create(fieldDecl->getInClassInitializer(), fieldDecl->getASTContext())}, m_classInitializerStyle{fieldDecl->getInClassInitStyle()}, m_hasDefaultMemberInitializer{fieldDecl->hasInClassInitializer()}, m_isMutable{fieldDecl->isMutable()}, m_isConst{fieldDecl->getType().isConstQualified()} @@ -2453,8 +2551,9 @@ class AstUsingDirective : public AstNode { UsingDirectiveDecl const* usingDirective, AzureClassesDatabase* const azureClassesDatabase, std::shared_ptr parentNode) - : AstNode(usingDirective), m_namedNamespace{usingDirective->getNominatedNamespaceAsWritten() - ->getQualifiedNameAsString()} + : AstNode(usingDirective), + m_namedNamespace{ + usingDirective->getNominatedNamespaceAsWritten()->getQualifiedNameAsString()} { azureClassesDatabase->CreateApiViewMessage( ApiViewMessages::UsingDirectiveFound, m_namedNamespace); @@ -2572,8 +2671,8 @@ class AstEnum : public AstNamedNode { : AstNamedNode(enumDecl, azureClassesDatabase, parentNode), m_underlyingType{enumDecl->getIntegerType().getAsString()}, m_isScoped{enumDecl->isScoped()}, m_isScopedWithClass{enumDecl->isScopedUsingClassTag()}, - m_isFixed{enumDecl->isFixed()}, m_isForwardDeclaration{ - enumDecl != enumDecl->getDefinition()} + m_isFixed{enumDecl->isFixed()}, + m_isForwardDeclaration{enumDecl != enumDecl->getDefinition()} { if (!m_isScoped) { diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp index 428940c92e1..5dd1cb16b1e 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp @@ -156,11 +156,25 @@ ApiViewProcessorImpl::ApiViewProcessorImpl( "Configuration element `filterNamespace` is neither a string or an array of strings."); } } - if (configurationJson.contains("additionalCompilerSwitches") - && configurationJson["additionalIncludeDirectories"].is_array() - && configurationJson["additionalIncludeDirectories"].size() != 0) + if (configurationJson.contains("additionalCompilerSwitches")) { - m_additionalCompilerArguments = configurationJson["additionalCompilerSwitches"]; + if (configurationJson["additionalCompilerSwitches"].is_array()) + { + if (configurationJson["additionalCompilerSwitches"].size() != 0) + { + m_additionalCompilerArguments = configurationJson["additionalCompilerSwitches"]; + } + } + else if (configurationJson["additionalCompilerSwitches"].is_string()) + { + m_additionalCompilerArguments.push_back( + configurationJson["additionalCompilerSwitches"].get()); + } + else if (!configurationJson["additionalCompilerSwitches"].is_null()) + { + throw std::runtime_error( + "Configuration element `additionalCompilerSwitches` is not an array or is empty."); + } } if (configurationJson.contains("additionalIncludeDirectories") && configurationJson["additionalIncludeDirectories"].is_array()) @@ -323,7 +337,10 @@ class ApiViewCompilationDatabase : public CompilationDatabase { "-c", "-std=c++14", "-Wall", - "-Werror"}; + "-Werror", + // Work around Microsoft STL requiring clang 16.0.0 or later. + "-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH", + }; public: ApiViewCompilationDatabase( @@ -331,8 +348,8 @@ class ApiViewCompilationDatabase : public CompilationDatabase { std::filesystem::path const& sourceLocation, std::vector const& additionalIncludePaths, std::vector const& additionalArguments) - : CompilationDatabase(), m_filesToCompile(filesToCompile), - m_sourceLocation(sourceLocation), m_additionalIncludePaths{additionalIncludePaths} + : CompilationDatabase(), m_filesToCompile(filesToCompile), m_sourceLocation(sourceLocation), + m_additionalIncludePaths{additionalIncludePaths} { for (auto const& arg : additionalArguments) { @@ -363,6 +380,7 @@ class ApiViewCompilationDatabase : public CompilationDatabase { { commandLine.push_back(arg); } + commandLine.push_back(std::string(stringFromU8string(file.u8string()))); std::vector rv; @@ -417,8 +435,6 @@ int ApiViewProcessorImpl::ProcessApiView() } // Create a compilation database consisting of the source root and source file. - auto absTemp = m_currentSourceRoot; - ApiViewCompilationDatabase compileDb( {std::filesystem::absolute(tempFile)}, m_currentSourceRoot, diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt b/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt index 5a576679b81..97a388c658c 100644 --- a/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt @@ -17,7 +17,9 @@ add_executable(parseTests TestCases/TemplateTests.cpp TestCases/ClassesWithInternalAndDetail.cpp TestCases/ExpressionTests.cpp - TestCases/UsingNamespace.cpp ) + TestCases/UsingNamespace.cpp + TestCases/DestructorTests.cpp +) add_dependencies(parseTests ApiViewProcessor) target_include_directories(parseTests PRIVATE ${ApiViewProcessor_SOURCE_DIR}) diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DestructorTests.cpp b/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DestructorTests.cpp new file mode 100644 index 00000000000..fe67555674e --- /dev/null +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DestructorTests.cpp @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +namespace Test { + +class BaseClassWithVirtualDestructor { + int* member{}; + +public: + virtual ~BaseClassWithVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class FinalBaseClassWithVirtualDestructor final { + int* member{}; + +public: + virtual ~FinalBaseClassWithVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class BaseClassWithNonVirtualDestructor { + int* member{}; + +public: + ~BaseClassWithNonVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class FinalBaseClassWithNonVirtualDestructor final { + int* member{}; + +public: + ~FinalBaseClassWithNonVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class BaseClassWithProtectedDestructor { + int* member{}; + +protected: + ~BaseClassWithProtectedDestructor() + { + if (member) + { + delete member; + } + }; +}; +class FinalBaseClassWithProtectedDestructor final { + int* member{}; + +protected: + ~FinalBaseClassWithProtectedDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class DerivedClassWithVirtualDestructor : public BaseClassWithVirtualDestructor { + int* member{}; + ~DerivedClassWithVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class DerivedClassWithNonVirtualDestructor : public BaseClassWithNonVirtualDestructor { + int* member{}; + ~DerivedClassWithNonVirtualDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class DerivedClassWithProtectedDestructor : public BaseClassWithProtectedDestructor { + int* member{}; + +protected: + virtual ~DerivedClassWithProtectedDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class FinalClassWithPrivateDestructor final { + int* member{}; + ~FinalClassWithPrivateDestructor() + { + if (member) + { + delete member; + } + }; +}; + +class ClassWithPrivateDestructor { + int* member{}; + ~ClassWithPrivateDestructor() + { + if (member) + { + delete member; + } + }; +}; + +} // namespace Test diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp b/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp index 085f56567ef..33cb79967d0 100644 --- a/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp @@ -108,7 +108,7 @@ class TestParser : public testing::Test { } } std::vector - defaultCommandLine{"clang++.exe", "-DAZ_RTTI", "-fcxx-exceptions", "-c", "-std=c++14"}; + defaultCommandLine{"clang++.exe", "-DAZ_RTTI", "-fcxx-exceptions", "-c", "-std=c++14", "-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH"}; // Inherited via CompilationDatabase virtual std::vector getCompileCommands(llvm::StringRef FilePath) const override { @@ -454,7 +454,7 @@ TEST_F(TestParser, Class1) NsDumper dumper; db->DumpClassDatabase(&dumper); - EXPECT_EQ(31ul, dumper.Messages.size()); + EXPECT_EQ(44ul, dumper.Messages.size()); size_t internalTypes = 0; for (const auto& msg : dumper.Messages) @@ -559,7 +559,7 @@ TEST_F(TestParser, UsingNamespace) size_t usingNamespaces = 0; for (const auto& msg : dumper.Messages) { - if (msg.DiagnosticId == "CPA0009") + if (msg.DiagnosticId == "CPA000A") { usingNamespaces += 1; } @@ -567,6 +567,42 @@ TEST_F(TestParser, UsingNamespace) EXPECT_EQ(usingNamespaces, 1ul); } +TEST_F(TestParser, TestDtors) +{ + ApiViewProcessor processor("tests", R"({ + "sourceFilesToProcess": [ + "DestructorTests.cpp" + ], + "additionalIncludeDirectories": [], + "additionalCompilerSwitches": null, + "allowInternal": false, + "includeDetail": false, + "includePrivate": false, + "filterNamespace": null +} +)"_json); + + EXPECT_EQ(processor.ProcessApiView(), 0); + + auto& db = processor.GetClassesDatabase(); + EXPECT_TRUE(SyntaxCheckClassDb(db, "DestructorTests1.cpp")); + + NsDumper dumper; + db->DumpClassDatabase(&dumper); + EXPECT_EQ(2ul, dumper.Messages.size()); + + size_t nonVirtualDestructor= 0; + for (const auto& msg : dumper.Messages) + { + if (msg.DiagnosticId == "CPA000B") + { + nonVirtualDestructor+= 1; + } + } + EXPECT_EQ(nonVirtualDestructor, 2ul); +} + + #if 0 TEST_F(TestParser, AzureCore1) { From 81aa85d367e27182ba4b126e5c3b294b9d7647f8 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 11 Sep 2023 14:14:33 -0400 Subject: [PATCH 08/93] Delete non compliant groups in playground subscription (#6929) --- eng/pipelines/live-test-cleanup.yml | 3 +-- eng/scripts/cleanup-allowlist.txt | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml index 4b0e41d7ec4..6b86821bca8 100644 --- a/eng/pipelines/live-test-cleanup.yml +++ b/eng/pipelines/live-test-cleanup.yml @@ -30,8 +30,7 @@ parameters: - DisplayName: AzureCloud Playground - Resource Cleanup SubscriptionConfigurations: - $(sub-config-azure-cloud-playground) - # TODO: Enable strict resource cleanup after pre-existing static groups have been handled - # AdditionalParameters: "-DeleteNonCompliantGroups" + AdditionalParameters: "-DeleteNonCompliantGroups" - DisplayName: Dogfood Translation - Resource Cleanup SubscriptionConfigurations: - $(sub-config-translation-int-test-resources) diff --git a/eng/scripts/cleanup-allowlist.txt b/eng/scripts/cleanup-allowlist.txt index 02705e9e235..dd65d9ff1ee 100644 --- a/eng/scripts/cleanup-allowlist.txt +++ b/eng/scripts/cleanup-allowlist.txt @@ -16,3 +16,5 @@ LiveTestSecrets # For example, Azure Synapse (synapseworkspace-managedrg-adaed0ea-df22-4ec1-9280-5ac85cf2f87a) and Azure Purview (managed-rg-mypurviewaccount). *managed* +# Azure Kubernetes Service node resource groups +MC_* From c875b4b3fe8bec05ca76bbbb291c539d41b93d34 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Tue, 12 Sep 2023 09:25:09 -0700 Subject: [PATCH 09/93] Update readme (#6947) --- .../python-packages/apiview-gpt/README.md | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/python-packages/apiview-gpt/README.md b/packages/python-packages/apiview-gpt/README.md index e24fa8f55a1..d51d8add2a9 100644 --- a/packages/python-packages/apiview-gpt/README.md +++ b/packages/python-packages/apiview-gpt/README.md @@ -19,9 +19,46 @@ To generate a review in JSON format: ``` OPENAI_API_BASE="" # The OpenAI endpoint URL OPENAI_API_KEY="" # The OpenAI API key +APIVIEW_GPT_SERVICE_URL=https://apiview-gpt.azurewebsites.net ``` -3. Run `python generate_review.py`. Right now it will just use `text.txt` as the input. -4. Output will be stored in `output.json`. +3. Run `python cli.py review generate --language --path ` +4. Output will be stored in `scratch\output\.json`. + +## Working With Semantic Documents + +APIView GPT uses semantic search to ground the ChatGPT request. To interact with the semantic database: +1. Add the following to your `.env` file: +``` +APIVIEW_API_KEY="" # The APIView API key +``` +2. Run `python cli.py vector --help` to see available commands. + +### Adding Semantic Documents + +To add new semantic documents, create a JSON file with the following format: +``` +{ + "language": "python", # can use any language APIView supports + "badCode": "def foo():\n pass\n", # must be a single line + "goodCode": "def foo():\n return None\n", # Optional. Must be a single line + "comment": "You should always have a return", # Optional. Free text comment about the bad code pattern + "guidelineIds": [ + "python_design.html#python-return-statements" # Optional. List of guideline IDs that this document is related to + ] +} +``` +The JSON file you can create should be a JSON array of one or more of the above document objects. Then run: +`python cli.py vector create --path ` + +The command will iterate through the JSON file and create a semantic document for each object. + +### Searching Semantic Documents + +Generating a review searches the semantic document database prior to sending the request to OpenAI. To search the semantic database directly (as in, for testing), +run the following command: +`python cli.py vector search --language --path ` + +Here code is a text file of the APIView you want to evaluate. ## Documentation From aa1eaf97ae39aea0db3a40507bfd574f00b644c9 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:37:03 -0700 Subject: [PATCH 10/93] allowing a bit more of a grace period on slow requests (#6918) --- tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs index 5508a54b2b9..663fd5560b5 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs @@ -160,6 +160,8 @@ private static void StartServer(StartOptions startOptions) .ConfigureKestrel(kestrelServerOptions => { kestrelServerOptions.ConfigureEndpointDefaults(lo => lo.Protocols = HttpProtocols.Http1); + // default minimum rate is 240 bytes per second with 5 second grace period. Bumping to 50bps with a graceperiod of 20 seconds. + kestrelServerOptions.Limits.MinRequestBodyDataRate = new MinDataRate(bytesPerSecond: 50, gracePeriod: TimeSpan.FromSeconds(20)); }) ); From fb6062a02ee9807feb9b1f716013b4c7cfebf7a8 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:25:31 -0700 Subject: [PATCH 11/93] Support Bulk Sanitizer Add (#6926) * add code and tests for bulk sanitizers * add docs reflecting new bulk sanitizer add --- .../AdminTests.cs | 119 ++++++++++++++++++ .../Azure.Sdk.Tools.TestProxy/Admin.cs | 33 ++++- .../Common/HttpRequestInteractions.cs | 27 ++++ .../Common/SanitizerList.cs | 10 ++ .../Azure.Sdk.Tools.TestProxy/README.md | 32 ++++- 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs index f52d8d72f52..843c590daef 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs @@ -3,6 +3,7 @@ using Azure.Sdk.Tools.TestProxy.Matchers; using Azure.Sdk.Tools.TestProxy.Sanitizers; using Azure.Sdk.Tools.TestProxy.Transforms; +using Microsoft.AspNetCore.DataProtection.KeyManagement; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging.Abstractions; @@ -11,6 +12,7 @@ using System.Linq; using System.Net; using System.Reflection; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Xunit; @@ -29,6 +31,123 @@ public class AdminTests { private NullLoggerFactory _nullLogger = new NullLoggerFactory(); + [Fact] + public async void TestAddSanitizersThrowsOnEmptyArray() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + var assertion = await Assert.ThrowsAsync( + async () => await controller.AddSanitizers() + ); + + assertion.StatusCode.Equals(HttpStatusCode.BadRequest); + } + + [Fact] + + public async void TestAddSanitizersHandlesPopulatedArray() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[ + { + ""Name"": ""GeneralRegexSanitizer"", + ""Body"": { + ""regex"": ""[a-zA-Z]?"", + ""value"": ""hello_there"", + ""condition"": { + ""UriRegex"": "".+/Tables"" + } + } + }, + { + ""Name"": ""HeaderRegexSanitizer"", + ""Body"": { + ""key"": ""Location"", + ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables"" + } + } +]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + await controller.AddSanitizers(); + + + Assert.Equal(2, testRecordingHandler.Sanitizers.Count); + + Assert.True(testRecordingHandler.Sanitizers[0] is GeneralRegexSanitizer); + Assert.True(testRecordingHandler.Sanitizers[1] is HeaderRegexSanitizer); + } + + [Fact] + public async void TestAddSanitizersThrowsOnSingleBadInput() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[ + { + ""Name"": ""GeneralRegexSanitizer"", + ""Body"": { + ""regex"": ""[a-zA-Z]?"", + ""value"": ""hello_there"", + ""condition"": { + ""UriRegex"": "".+/Tables"" + } + } + }, + { + ""Name"": ""BadRegexIdentifier"", + ""Body"": { + ""key"": ""Location"", + ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables"" + } + } +]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + var assertion = await Assert.ThrowsAsync( + async () => await controller.AddSanitizers() + ); + + assertion.StatusCode.Equals(HttpStatusCode.BadRequest); + } + [Fact] public async void TestAddSanitizerThrowsOnInvalidAbstractionId() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs index d81b7ce8588..f8d2f1ec7d3 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Azure.Sdk.Tools.TestProxy.Common; @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Tasks; @@ -81,6 +82,36 @@ public async Task AddSanitizer() } } + [HttpPost] + public async Task AddSanitizers() + { + DebugLogger.LogAdminRequestDetails(_logger, Request); + var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true); + + // parse all of them first, any exceptions should pop here + var workload = (await HttpRequestInteractions.GetBody>(Request)).Select(s => (RecordedTestSanitizer)GetSanitizer(s.Name, s.Body)).ToList(); + + if (workload.Count == 0) + { + throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items."); + } + + // register them all + foreach(var sanitizer in workload) + { + if (recordingId != null) + { + _recordingHandler.AddSanitizerToRecording(recordingId, sanitizer); + } + else + { + _recordingHandler.Sanitizers.Add(sanitizer); + } + } + + } + + [HttpPost] public async Task SetMatcher() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs index 7de8a0a04e1..60bd3ae8164 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs @@ -4,6 +4,7 @@ using Azure.Core; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using System; using System.IO; @@ -61,6 +62,32 @@ public static string GetBodyKey(JsonDocument document, string key, bool allowNul return value; } + public async static Task GetBody(HttpRequest req) + { + if (req.ContentLength > 0) + { + try + { + using (var jsonDocument = await JsonDocument.ParseAsync(req.Body, options: new JsonDocumentOptions() { AllowTrailingCommas = true })) + { + return JsonSerializer.Deserialize(jsonDocument.RootElement.GetRawText(), new JsonSerializerOptions() { }); + } + + } + catch (Exception e) + { + req.Body.Position = 0; + using (StreamReader readstream = new StreamReader(req.Body, Encoding.UTF8)) + { + string bodyContent = readstream.ReadToEnd(); + throw new HttpException(HttpStatusCode.BadRequest, $"The body of this request is invalid JSON. Content: {bodyContent}. Exception detail: {e.Message}"); + } + } + } + + return default(T); + } + public async static Task GetBody(HttpRequest req) { if (req.ContentLength > 0) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs new file mode 100644 index 00000000000..fe8dc075db0 --- /dev/null +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace Azure.Sdk.Tools.TestProxy.Common +{ + public class SanitizerBody { + public string Name { get; set; } + public JsonDocument Body { get; set; } + } +} diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index fcadbc371d8..f2334f1682d 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -524,7 +524,7 @@ Add a more expansive Header sanitizer that uses a target group instead of filter ```jsonc // POST to URI /Admin/AddSanitizer -// dictionary dictionary +// headers { "x-abstraction-identifier": "HeaderRegexSanitizer" } @@ -549,6 +549,36 @@ Each sanitizer is optionally prefaced with the **specific part** of the request/ A sanitizer that does _not_ include this prefix is something different, and probably applies at the session level instead on an individual request/response pair. +#### Passing sanitizers in bulk + +In some cases, users need to register a lot (10+) of sanitizers. In this case, going back and forth with the proxy server individually is not very efficient. To ameliorate this, the proxy honors multiple sanitizers in the same request if the user utilizes `/Admin/AddSanitizers`. + +```jsonc +// POST to URI /Admin/AddSanitizers +// note the request body is simply an array of objects +[ + { + "Name": "GeneralRegexSanitizer", + "Body": { + "regex": "[a-zA-Z]?", + "value": "hello_there", + "condition": { + "UriRegex": ".+/Tables" + } + } + }, + { + "Name": "HeaderRegexSanitizer", + "Body": { // <-- the contents of this property mirror what would be passed in the request body for individual AddSanitizer() + "key": "Location", + "value": "fakeaccount", + "regex": "https\\:\\/\\/(?[a-z]+)\\.(?:table|blob|queue)\\.core\\.windows\\.net", + "groupForReplace": "account" + } + } +] +``` + ### For Sanitizers, Matchers, or Transforms in general When invoked as basic requests to the `Admin` controller, these settings will be applied to **all** further requests and responses. Both `Playback` and `Recording`. Where applicable. From c3f571ba49ab84636a5df0cf6a3641b73dce5369 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:44:19 -0700 Subject: [PATCH 12/93] Force entire `Restore` operation to enqueued (#6948) * force the entire restore queue to a semaphore operation * we must ensure to use the proper Enqueue method! --- .../Store/GitStore.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs index a94dfde87f4..34d0f913f4c 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs @@ -172,15 +172,19 @@ public async Task Push(string pathToAssetsJson) { public async Task Restore(string pathToAssetsJson) { var config = await ParseConfigurationFile(pathToAssetsJson); - var initialized = IsAssetsRepoInitialized(config); - - if (!initialized) + var restoreQueue = InitTasks.GetOrAdd("restore", new TaskQueue()); + await restoreQueue.EnqueueAsync(async () => { - InitializeAssetsRepo(config); - } + var initialized = IsAssetsRepoInitialized(config); - CheckoutRepoAtConfig(config, cleanEnabled: true); - await BreadCrumb.Update(config); + if (!initialized) + { + InitializeAssetsRepo(config); + } + + CheckoutRepoAtConfig(config, cleanEnabled: true); + await BreadCrumb.Update(config); + }); return config.AssetsRepoLocation.ToString(); } From 52384101c2ec432a22217380e6ef7d8ccf530a1e Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 12 Sep 2023 16:58:55 -0400 Subject: [PATCH 13/93] Skip cleaning up locked resource groups (#6951) --- eng/scripts/live-test-resource-cleanup.ps1 | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index 79dcc37a728..502c044fa4e 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -309,7 +309,12 @@ function HasDoNotDeleteTag([object]$ResourceGroup) { return $doNotDelete -ne $null } -function HasDeleteLock() { +function HasDeleteLock([object]$ResourceGroup) { + $lock = Get-AzResourceLock -ResourceGroupName $ResourceGroup.ResourceGroupName + if ($lock) { + Write-Host " Skipping locked resource group '$($ResourceGroup.ResourceGroupName)'" + return $true + } return $false } @@ -347,7 +352,10 @@ function DeleteOrUpdateResourceGroups() { if (HasValidAliasInName $rg) { continue } - if (HasValidOwnerTag $rg -or HasDeleteLock $rg) { + if (HasValidOwnerTag $rg) { + continue + } + if (HasDeleteLock $rg) { continue } $toUpdate += $rg From 88cfc5c84344f56d27d6bb5623562f2d8dd67e37 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 12 Sep 2023 17:00:42 -0400 Subject: [PATCH 14/93] [stress] Add maintenance window and enable node security upgrades (#6917) --- .../cluster/azure/cluster/cluster.bicep | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep index 73ea6f6c120..84ec0aafecc 100644 --- a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep +++ b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep @@ -53,7 +53,7 @@ var agentPools = [ defaultAgentPool ] -resource newCluster 'Microsoft.ContainerService/managedClusters@2022-09-02-preview' = if (!updateNodes) { +resource newCluster 'Microsoft.ContainerService/managedClusters@2023-02-02-preview' = if (!updateNodes) { name: clusterName location: location tags: tags @@ -75,6 +75,10 @@ resource newCluster 'Microsoft.ContainerService/managedClusters@2022-09-02-previ enabled: true } } + autoUpgradeProfile: { + nodeOSUpgradeChannel: 'SecurityPatch' + upgradeChannel: null + } kubernetesVersion: kubernetesVersion enableRBAC: true dnsPrefix: dnsPrefix @@ -86,7 +90,26 @@ resource newCluster 'Microsoft.ContainerService/managedClusters@2022-09-02-previ } } -resource existingCluster 'Microsoft.ContainerService/managedClusters@2022-09-02-preview' existing = if (updateNodes) { +resource maintenanceConfig 'Microsoft.ContainerService/managedClusters/maintenanceConfigurations@2023-05-02-preview' = if (!updateNodes) { + name: 'aksManagedNodeOSUpgradeSchedule' + parent: newCluster + properties: { + maintenanceWindow: { + durationHours: 4 + utcOffset: '-08:00' + startTime: '02:00' + schedule: { + weekly: { + dayOfWeek: 'Monday' + intervalWeeks: 1 + } + } + } + } +} + + +resource existingCluster 'Microsoft.ContainerService/managedClusters@2023-02-02-preview' existing = if (updateNodes) { name: clusterName } From 33086ab3b82d18c7a20761b503e033b71246a1a4 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:01:22 -0700 Subject: [PATCH 15/93] enforce each restore path to its own workqueue (#6955) --- tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs index 34d0f913f4c..95234d4b7dc 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs @@ -172,7 +172,8 @@ public async Task Push(string pathToAssetsJson) { public async Task Restore(string pathToAssetsJson) { var config = await ParseConfigurationFile(pathToAssetsJson); - var restoreQueue = InitTasks.GetOrAdd("restore", new TaskQueue()); + var restoreQueue = InitTasks.GetOrAdd(config.AssetsJsonRelativeLocation, new TaskQueue()); + await restoreQueue.EnqueueAsync(async () => { var initialized = IsAssetsRepoInitialized(config); From eaa8801580482c9f07ad4977103f2abadbc45d99 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Wed, 13 Sep 2023 09:47:19 -0700 Subject: [PATCH 16/93] Refactor (#6965) --- packages/python-packages/apiview-gpt/cli.py | 5 ++- .../{acr.txt => python/acr_clean.txt} | 31 +++++++++++++ .../apiviews/{test.txt => python/noodle.txt} | 27 +++++++++--- .../apiview-gpt/scratch/apiviews/test2.txt | 1 - .../comments/{comments.json => python.json} | 0 .../apiview-gpt/scratch/output/acr.json | 12 ----- .../scratch/output/python/acr_clean.json | 4 ++ .../scratch/output/python/noodle.json | 44 +++++++++++++++++++ .../apiview-gpt/scratch/output/test.json | 42 ------------------ 9 files changed, 103 insertions(+), 63 deletions(-) rename packages/python-packages/apiview-gpt/scratch/apiviews/{acr.txt => python/acr_clean.txt} (99%) rename packages/python-packages/apiview-gpt/scratch/apiviews/{test.txt => python/noodle.txt} (61%) delete mode 100644 packages/python-packages/apiview-gpt/scratch/apiviews/test2.txt rename packages/python-packages/apiview-gpt/scratch/comments/{comments.json => python.json} (100%) delete mode 100644 packages/python-packages/apiview-gpt/scratch/output/acr.json create mode 100644 packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json create mode 100644 packages/python-packages/apiview-gpt/scratch/output/python/noodle.json delete mode 100644 packages/python-packages/apiview-gpt/scratch/output/test.json diff --git a/packages/python-packages/apiview-gpt/cli.py b/packages/python-packages/apiview-gpt/cli.py index a407baea77c..b3a58bb0456 100644 --- a/packages/python-packages/apiview-gpt/cli.py +++ b/packages/python-packages/apiview-gpt/cli.py @@ -82,7 +82,10 @@ def generate_review(language: str, path: str): with open(path, "r") as f: apiview = f.read() review = rg.get_response(apiview, language) - with open(os.path.join('scratch', 'output', f'{filename}.json'), 'w') as f: + output_path = os.path.join('scratch', 'output', language) + if not os.path.exists(output_path): + os.makedirs(output_path) + with open(os.path.join(output_path, f'{filename}.json'), 'w') as f: f.write(review.json(indent=4)) pprint(review) diff --git a/packages/python-packages/apiview-gpt/scratch/apiviews/acr.txt b/packages/python-packages/apiview-gpt/scratch/apiviews/python/acr_clean.txt similarity index 99% rename from packages/python-packages/apiview-gpt/scratch/apiviews/acr.txt rename to packages/python-packages/apiview-gpt/scratch/apiviews/python/acr_clean.txt index 8b0d9a8db8c..1a455cfff6d 100644 --- a/packages/python-packages/apiview-gpt/scratch/apiviews/acr.txt +++ b/packages/python-packages/apiview-gpt/scratch/apiviews/python/acr_clean.txt @@ -13,10 +13,12 @@ class azure.containerregistry.ArtifactArchitecture(str, Enum): RISCV64 = "riscv64" S390X = "s390x" WASM = "wasm" + class azure.containerregistry.ArtifactManifestOrder(str, Enum): LAST_UPDATED_ON_ASCENDING = "timeasc" LAST_UPDATED_ON_DESCENDING = "timedesc" NONE = "none" + class azure.containerregistry.ArtifactManifestProperties: property architecture: ArtifactArchitecture # Read-only property created_on: datetime # Read-only @@ -40,6 +42,7 @@ class azure.containerregistry.ArtifactManifestProperties: ivar size_in_bytes: str ivar tags: List[str] def __init__(self, **kwargs) + class azure.containerregistry.ArtifactOperatingSystem(str, Enum): AIX = "aix" ANDROID = "android" @@ -55,10 +58,12 @@ class azure.containerregistry.ArtifactOperatingSystem(str, Enum): PLAN9 = "plan9" SOLARIS = "solaris" WINDOWS = "windows" + class azure.containerregistry.ArtifactTagOrder(str, Enum): LAST_UPDATED_ON_ASCENDING = "timeasc" LAST_UPDATED_ON_DESCENDING = "timedesc" NONE = "none" + class azure.containerregistry.ArtifactTagProperties: property created_on: datetime # Read-only property digest: str # Read-only @@ -75,7 +80,9 @@ class azure.containerregistry.ArtifactTagProperties: ivar name: str ivar repository: str def __init__(self, **kwargs) + class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClient): implements ContextManager + def __init__( self, endpoint: str, @@ -85,7 +92,9 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien audience: str = ..., **kwargs ) -> None + def close(self) -> None + @distributed_trace def delete_blob( self, @@ -93,6 +102,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag_or_digest: str, **kwargs ) -> None + @distributed_trace def delete_manifest( self, @@ -100,12 +110,14 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag_or_digest: str, **kwargs ) -> None + @distributed_trace def delete_repository( self, repository: str, **kwargs ) -> None + @distributed_trace def delete_tag( self, @@ -113,6 +125,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag: str, **kwargs ) -> None + @distributed_trace def download_blob( self, @@ -120,6 +133,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien digest: str, **kwargs ) -> DownloadBlobResult | None + @distributed_trace def download_manifest( self, @@ -127,6 +141,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag_or_digest: str, **kwargs ) -> DownloadManifestResult + @distributed_trace def get_manifest_properties( self, @@ -134,12 +149,14 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag_or_digest: str, **kwargs ) -> ArtifactManifestProperties + @distributed_trace def get_repository_properties( self, repository: str, **kwargs ) -> RepositoryProperties + @distributed_trace def get_tag_properties( self, @@ -147,6 +164,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag: str, **kwargs ) -> ArtifactTagProperties + @distributed_trace def list_manifest_properties( self, @@ -156,6 +174,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien results_per_page: int = ..., **kwargs ) -> ItemPaged[ArtifactManifestProperties] + @distributed_trace def list_repository_names( self, @@ -163,6 +182,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien results_per_page: int = ..., **kwargs ) -> ItemPaged[str] + @distributed_trace def list_tag_properties( self, @@ -172,6 +192,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien results_per_page: int = ..., **kwargs ) -> ItemPaged[ArtifactTagProperties] + @overload def update_manifest_properties( self, @@ -180,6 +201,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien properties, **kwargs ) -> ArtifactManifestProperties + @overload def update_manifest_properties( self, @@ -187,6 +209,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag_or_digest, **kwargs ) -> ArtifactManifestProperties + @distributed_trace def update_manifest_properties( self, @@ -198,6 +221,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien can_write: Optional[bool] = ..., **kwargs ) -> ArtifactManifestProperties + @overload def update_repository_properties( self, @@ -205,12 +229,14 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien properties, **kwargs ) -> RepositoryProperties + @overload def update_repository_properties( self, repository, **kwargs ) -> RepositoryProperties + @distributed_trace def update_repository_properties( self, @@ -222,6 +248,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien can_write: Optional[bool] = ..., **kwargs ) -> RepositoryProperties + @overload def update_tag_properties( self, @@ -230,6 +257,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien properties, **kwargs ) -> ArtifactTagProperties + @overload def update_tag_properties( self, @@ -237,6 +265,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien tag, **kwargs ) -> ArtifactTagProperties + @distributed_trace def update_tag_properties( self, @@ -249,6 +278,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien **kwargs ) -> ArtifactTagProperties + @distributed_trace def upload_blob( self, @@ -257,6 +287,7 @@ class azure.containerregistry.ContainerRegistryClient(ContainerRegistryBaseClien **kwargs ) -> str + @distributed_trace def upload_manifest( self, diff --git a/packages/python-packages/apiview-gpt/scratch/apiviews/test.txt b/packages/python-packages/apiview-gpt/scratch/apiviews/python/noodle.txt similarity index 61% rename from packages/python-packages/apiview-gpt/scratch/apiviews/test.txt rename to packages/python-packages/apiview-gpt/scratch/apiviews/python/noodle.txt index abbe5a3dbfe..c2a3f8a167b 100644 --- a/packages/python-packages/apiview-gpt/scratch/apiviews/test.txt +++ b/packages/python-packages/apiview-gpt/scratch/apiviews/python/noodle.txt @@ -1,20 +1,31 @@ # Package is parsed using apiview-stub-generator(version:0.3.7), Python version: 3.10.12 +class azure.contoso.NoodleCreateRequest: + + ivar name: str + ivar color: NoodleColor + + def __init__( + self, + name: str, + color: Optional[NoodleColor] + ) + class azure.contoso.NoodleColor(Enum): blue = "blue" green = "green" red = "red" -class azure.contoso.NoodleResult: +class azure.contoso.NoodleResponse: ivar name: str - ivar color: NoodleColor + ivar color: Optional[NoodleColor] def __init__( self, name: str, - color: NoodleColor + color: Optional[NoodleColor] ) class azure.contoso.NoodleAsyncManager: @@ -26,9 +37,9 @@ class azure.contoso.NoodleAsyncManager: options: dict ) - async def get_noodle_async(self, options: dict) -> NoodleResult + async def get_noodle_async(self, options: dict) -> NoodleResponse - async def get_noodles_async(self, options: dict) -> List[NoodleResult] + async def get_noodles_async(self, options: dict) -> List[NoodleResponse] class azure.contoso.NoodleManager: @@ -40,6 +51,8 @@ class azure.contoso.NoodleManager: options: dict ) - def get_noodle(self, options: dict) -> NoodleResult + def create_noodle(self, body: NoodleCreateRequest, **kwargs) -> NoodleResponse + + def get_noodle(self, options: dict) -> NoodleResponse - def list_noodles(self, options: dict) -> List[NoodleResult] + def get_noodles(self, options: dict) -> List[NoodleResponse] diff --git a/packages/python-packages/apiview-gpt/scratch/apiviews/test2.txt b/packages/python-packages/apiview-gpt/scratch/apiviews/test2.txt deleted file mode 100644 index 04bd42ad10b..00000000000 --- a/packages/python-packages/apiview-gpt/scratch/apiviews/test2.txt +++ /dev/null @@ -1 +0,0 @@ -# Package is parsed using apiview-stub-generator(version:0.3.7), Python version: 3.10.12\n\nclass azure.contoso.ContosoAsyncManager:\n\n async def __init__(\n self,\n endpoint,\n credential,\n options: dict\n )\n\n async def get_widget_async(self, options: dict) -> WidgetResult\n\nclass azure.contoso.ContosoManager:\n\n def __init__(\n self,\n endpoint,\n credential,\n options: dict\n )\n\n def get_widget(self, options: dict) -> WidgetResult\n\nclass azure.contoso.WidgetColor(Enum):\n\n blue = "blue"\n green = "green"\n red = "red"\n\nclass azure.contoso.WidgetResult:\n\n def __init__(\n self,\n name: str,\n color: WidgetColor\n )\n\n \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/comments/comments.json b/packages/python-packages/apiview-gpt/scratch/comments/python.json similarity index 100% rename from packages/python-packages/apiview-gpt/scratch/comments/comments.json rename to packages/python-packages/apiview-gpt/scratch/comments/python.json diff --git a/packages/python-packages/apiview-gpt/scratch/output/acr.json b/packages/python-packages/apiview-gpt/scratch/output/acr.json deleted file mode 100644 index fcdda52b36c..00000000000 --- a/packages/python-packages/apiview-gpt/scratch/output/acr.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "status": "Error", - "violations": [ - { - "rule_ids": [], - "line_no": 269, - "bad_code": "class azure.containerregistry.RepositoryProperties:", - "suggestion": "class azure.containerregistry.Repository:", - "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'Repository' instead." - } - ] -} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json new file mode 100644 index 00000000000..d6c71837344 --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json @@ -0,0 +1,4 @@ +{ + "status": "Success", + "violations": [] +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json new file mode 100644 index 00000000000..79458428638 --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json @@ -0,0 +1,44 @@ +{ + "status": "Error", + "violations": [ + { + "rule_ids": [ + "python_design.html#python-client-naming", + "python_design.html#python-namespaces-async", + "python_design.html#python-client-separate-sync-async" + ], + "line_no": 30, + "bad_code": "class azure.contoso.NoodleAsyncManager:", + "suggestion": "Separate client classes for synchronous and asynchronous operations.", + "comment": "Service client types should be named with a Client suffix. Use an .aio suffix added to the namespace of the sync client for async clients. Do not combine async and sync operations in the same class." + }, + { + "rule_ids": [ + "python_design.html#python-client-constructor-policy-arguments" + ], + "line_no": 31, + "bad_code": "async def __init__(self, endpoint, credential, options: dict)", + "suggestion": "async def __init__(self, endpoint, credential, **kwargs)", + "comment": "Accept optional default request options as keyword arguments and pass them along to its pipeline policies." + }, + { + "rule_ids": [ + "python_design.html#python-client-naming", + "python_design.html#python-client-separate-sync-async" + ], + "line_no": 43, + "bad_code": "class azure.contoso.NoodleManager:", + "suggestion": "Provide two separate client classes for synchronous and asynchronous operations.", + "comment": "The class name should have a 'Client' suffix. Do not combine async and sync operations in the same class." + }, + { + "rule_ids": [ + "python_design.html#python-client-connection-string" + ], + "line_no": 49, + "bad_code": "connection_string: Optional[str],", + "suggestion": "Remove the connection_string parameter from the constructor and create a separate factory classmethod from_connection_string to create a client from a connection string.", + "comment": "The constructor (__init__ method) must not take a connection string." + } + ] +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/test.json b/packages/python-packages/apiview-gpt/scratch/output/test.json deleted file mode 100644 index 17487f2283e..00000000000 --- a/packages/python-packages/apiview-gpt/scratch/output/test.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "status": "Error", - "violations": [ - { - "rule_ids": [ - "python_design.html#python-client-naming", - "python_design.html#python-namespaces-async" - ], - "line_no": 19, - "bad_code": "class azure.contoso.NoodleAsyncManager:", - "suggestion": "class azure.contoso.aio.NoodleAsyncClient:", - "comment": "The class name should end with a 'Client' suffix. The class should be in a namespace with an '.aio' suffix." - }, - { - "rule_ids": [ - "python_design.html#python-client-constructor-policy-arguments" - ], - "line_no": 20, - "bad_code": "async def __init__(self, endpoint, credential, options: dict)", - "suggestion": "async def __init__(self, endpoint, credential, **kwargs)", - "comment": "The class should accept optional default request options as keyword arguments and pass them along to its pipeline policies." - }, - { - "rule_ids": [ - "python_design.html#python-client-naming" - ], - "line_no": 32, - "bad_code": "class azure.contoso.NoodleManager:", - "suggestion": "class azure.contoso.NoodleClient:", - "comment": "The class name should end with a 'Client' suffix." - }, - { - "rule_ids": [ - "python_design.html#python-client-connection-string" - ], - "line_no": 38, - "bad_code": "connection_string: Optional[str],", - "suggestion": "Implement a separate factory classmethod from_connection_string to create a client from a connection string.", - "comment": "The constructor (__init__ method) should not take a connection string as an argument." - } - ] -} \ No newline at end of file From f73798cc417adb8bfc665c09f85166c3ac9ad314 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 13 Sep 2023 14:18:22 -0400 Subject: [PATCH 17/93] Skip cleaning up child resource groups (#6966) --- eng/scripts/live-test-resource-cleanup.ps1 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index 502c044fa4e..2f6a0a8669a 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -309,6 +309,14 @@ function HasDoNotDeleteTag([object]$ResourceGroup) { return $doNotDelete -ne $null } +function IsChildResource([object]$ResourceGroup) { + if ($ResourceGroup.ManagedBy) { + Write-Host " Skipping resource group '$($ResourceGroup.ResourceGroupName)' because it is managed by '$($ResourceGroup.ManagedBy)'" + return $true + } + return $false +} + function HasDeleteLock([object]$ResourceGroup) { $lock = Get-AzResourceLock -ResourceGroupName $ResourceGroup.ResourceGroupName if ($lock) { @@ -349,6 +357,9 @@ function DeleteOrUpdateResourceGroups() { if (HasDoNotDeleteTag $rg) { continue } + if (IsChildResource $rg) { + continue + } if (HasValidAliasInName $rg) { continue } From e2bcb038b38205e16f6b7bf0050347c5ec34a2bc Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 13 Sep 2023 13:47:52 -0700 Subject: [PATCH 18/93] Add legacy moniker migration logic (#6895) * Add legacy moniker migration logic * Add functionality from Java testing --- .../Service-Level-Readme-Automation.ps1 | 8 +- .../scripts/Update-DocsMsPackageMonikers.ps1 | 122 ++++++++++++++++++ eng/common/scripts/common.ps1 | 1 + 3 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 eng/common/scripts/Update-DocsMsPackageMonikers.ps1 diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index e7dcbf7bf5f..a03e78e4e22 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -40,7 +40,10 @@ param( [string]$ClientSecret, [Parameter(Mandatory = $false)] - [string]$ReadmeFolderRoot = "docs-ref-services" + [string]$ReadmeFolderRoot = "docs-ref-services", + + [Parameter(Mandatory = $false)] + [array]$Monikers = @('latest', 'preview', 'legacy') ) . $PSScriptRoot/common.ps1 . $PSScriptRoot/Helpers/Service-Level-Readme-Automation-Helpers.ps1 @@ -50,8 +53,7 @@ param( Set-StrictMode -Version 3 $fullMetadata = Get-CSVMetadata -$monikers = @("latest", "preview") -foreach($moniker in $monikers) { +foreach($moniker in $Monikers) { # The onboarded packages return is key-value pair, which key is the package index, and value is the package info from {metadata}.json # E.g. # Key as: @azure/storage-blob diff --git a/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 new file mode 100644 index 00000000000..f1f282a6b37 --- /dev/null +++ b/eng/common/scripts/Update-DocsMsPackageMonikers.ps1 @@ -0,0 +1,122 @@ +<# +.SYNOPSIS +Move metadata JSON and package-level overview markdown files for deprecated packages to the legacy folder. + +.DESCRIPTION +Move onboarding information to the "legacy" moniker for whose support is "deprecated" in the Metadata CSV. +Only one version of a package can be documented in the "legacy" moniker. If multiple versions are available, +the "latest" version will be used and the "preview" version will be deleted. + +.PARAMETER DocRepoLocation +The location of the target docs repository. +#> + +param( + [Parameter(Mandatory = $true)] + [string] $DocRepoLocation +) + +. (Join-Path $PSScriptRoot common.ps1) + +Set-StrictMode -Version 3 + +function getPackageMetadata($moniker) { + $jsonFiles = Get-ChildItem -Path (Join-Path $DocRepoLocation "metadata/$moniker") -Filter *.json + $metadata = @{} + + foreach ($jsonFile in $jsonFiles) { + $packageMetadata = Get-Content $jsonFile -Raw | ConvertFrom-Json -AsHashtable + $packageIdentity = $packageMetadata.Name + if (Test-Path "Function:$GetPackageIdentity") { + $packageIdentity = &$GetPackageIdentity $packageMetadata + } + + $metadata[$packageIdentity] = @{ File = $jsonFile; Metadata = $packageMetadata } + } + + return $metadata +} + +function getPackageInfoFromLookup($packageIdentity, $version, $lookupTable) { + if ($lookupTable.ContainsKey($packageIdentity)) { + if ($lookupTable[$packageIdentity]['Metadata'].Version -eq $version) { + # Only return if the version matches + return $lookupTable[$packageIdentity] + } + } + + return $null +} + +function moveToLegacy($packageInfo) { + $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] + + Write-Host "Move to legacy: $($packageInfo['Metadata'].Name)" + $packageInfoPath = $packageInfo['File'] + Move-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" "$DocRepoLocation/metadata/legacy/" -Force + + $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" + if (Test-Path $readmePath) { + Move-Item ` + $readmePath ` + "$DocRepoLocation/$($docsMsMetadata.LegacyReadMeLocation)/" ` + -Force + } +} + +function deletePackageInfo($packageInfo) { + $docsMsMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageInfo['Metadata'] + + Write-Host "Delete superseded package: $($packageInfo['Metadata'].Name)" + $packageInfoPath = $packageInfo['File'] + Remove-Item "$($packageInfoPath.Directory)/$($packageInfoPath.BaseName).*" -Force + + $readmePath = "$DocRepoLocation/$($docsMsMetadata.PreviewReadMeLocation)/$($docsMsMetadata.DocsMsReadMeName)-readme.md" + if (Test-Path $readmePath) { + Remove-Item $readmePath -Force + } +} + +$metadataLookup = @{ + 'latest' = getPackageMetadata 'latest' + 'preview' = getPackageMetadata 'preview' +} +$deprecatedPackages = (Get-CSVMetadata).Where({ $_.Support -eq 'deprecated' }) + +foreach ($package in $deprecatedPackages) { + $packageIdentity = $package.Package + if (Test-Path "Function:$GetPackageIdentityFromCsvMetadata") { + $packageIdentity = &$GetPackageIdentityFromCsvMetadata $package + } + + $packageInfoPreview = $packageInfoLatest = $null + if ($package.VersionPreview) { + $packageInfoPreview = getPackageInfoFromLookup ` + -packageIdentity $packageIdentity ` + -version $package.VersionPreview ` + -lookupTable $metadataLookup['preview'] + } + + if ($package.VersionGA) { + $packageInfoLatest = getPackageInfoFromLookup ` + -packageIdentity $packageIdentity ` + -version $package.VersionGA ` + -lookupTable $metadataLookup['latest'] + } + + if (!$packageInfoPreview -and !$packageInfoLatest) { + # Nothing to move or delete + continue + } + + if ($packageInfoPreview -and $packageInfoLatest) { + # Delete metadata JSON and package-level overview markdown files for + # the preview version instead of moving both. This mitigates situations + # where the "latest" verison doesn't have a package-level overview + # markdown file and the "preview" version does. + deletePackageInfo $packageInfoPreview + moveToLegacy $packageInfoLatest + } else { + moveToLegacy ($packageInfoPreview ?? $packageInfoLatest) + } +} diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index e6b5f09fe7c..39d65cdd681 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -67,3 +67,4 @@ $GetEmitterPackageLockPathFn = "Get-${Language}-EmitterPackageLockPath" $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding" $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded" $GetPackageIdentity = "Get-${Language}-PackageIdentity" +$GetPackageIdentityFromCsvMetadata = "Get-${Language}-PackageIdentityFromCsvMetadata" From 7cedd65dfc94fef8bf37023087e46989db123505 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Wed, 13 Sep 2023 14:37:31 -0700 Subject: [PATCH 19/93] Add support for logging request prompts. (#6968) --- .../python-packages/apiview-gpt/.gitignore | 2 + packages/python-packages/apiview-gpt/cli.py | 16 +++++--- .../apiview-gpt/scratch/comments/python.json | 21 +++++++++- .../scratch/output/python/noodle.json | 20 ++++----- .../apiview-gpt/src/_gpt_reviewer.py | 41 +++++++++++++++++-- 5 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 packages/python-packages/apiview-gpt/.gitignore diff --git a/packages/python-packages/apiview-gpt/.gitignore b/packages/python-packages/apiview-gpt/.gitignore new file mode 100644 index 00000000000..7c35925c4cb --- /dev/null +++ b/packages/python-packages/apiview-gpt/.gitignore @@ -0,0 +1,2 @@ +prompts +raw_requests \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/cli.py b/packages/python-packages/apiview-gpt/cli.py index b3a58bb0456..17111897453 100644 --- a/packages/python-packages/apiview-gpt/cli.py +++ b/packages/python-packages/apiview-gpt/cli.py @@ -59,7 +59,7 @@ def delete_document(document_id: str): db.delete_document(document_id) print(f"Deleted document {document_id}") -def search_documents(language: str, path: str): +def search_documents(language: str, path: str, log_result: bool = False): """ Search for vector documents by similarity """ @@ -67,16 +67,17 @@ def search_documents(language: str, path: str): with open(path, "r") as f: code = f.read() results = de.search_documents(language, code) - with open('results.json', 'w') as f: - json.dump(results, f, indent=4) + if log_result: + with open('search_result_dump.json', 'w') as f: + json.dump(results, f, indent=4) pprint(results) -def generate_review(language: str, path: str): +def generate_review(language: str, path: str, log_prompt: bool = False): """ Generate a review for an APIView """ from src import GptReviewer - rg = GptReviewer() + rg = GptReviewer(log_prompt=log_prompt) filename = os.path.splitext(os.path.basename(path))[0] with open(path, "r") as f: @@ -116,13 +117,16 @@ def load_command_table(self, args): return OrderedDict(self.command_table) def load_arguments(self, command): + with ArgumentsContext(self, "") as ac: + ac.argument("language", type=str, help="The language of the APIView file", options_list=("--language", "-l")) with ArgumentsContext(self, "vector") as ac: ac.argument("document_id", type=str, help="The ID of the document to retrieve", options_list=("--id")) + ac.argument("log_result", action="store_true", help="Log the search results to a file called 'search_result_dump.json'") with ArgumentsContext(self, "guidelines") as ac: ac.argument("path", type, help="The path to the guidelines") with ArgumentsContext(self, "review") as ac: ac.argument("path", type=str, help="The path to the APIView file") - ac.argument("language", type=str, help="The language of the APIView file") + ac.argument("log_prompt", action="store_true", help="Log the prompt to a file called 'prompt.json'") super(CliCommandsLoader, self).load_arguments(command) diff --git a/packages/python-packages/apiview-gpt/scratch/comments/python.json b/packages/python-packages/apiview-gpt/scratch/comments/python.json index a8eac0be97f..16a08211578 100644 --- a/packages/python-packages/apiview-gpt/scratch/comments/python.json +++ b/packages/python-packages/apiview-gpt/scratch/comments/python.json @@ -78,6 +78,23 @@ "language": "python", "badCode": "def dump(self, dest: Union[str, PathLike, IO[AnyStr]], *, kwargs: dict = ..., **kwargs) -> None", "comment": "Don't use kwargs and **kwargs in the same signature. All individual kwarg values should be listed (except for private/internal ones)" - - } + }, + { + "language": "python", + "badCode": "def get_cats(self, **kwargs) -> List[Cat]", + "goodCode": "def list_cats(self, **kwargs) -> List[Cat]", + "comment": "The 'get_' prefix is for a single item. Use 'list_' for a collection." + }, + { + "language": "python", + "badCode": "def get_cat(self, **kwargs) -> GetCatResponse", + "goodCode": "def get_cat(self, **kwargs) -> Cat", + "comment": "Avoid using terms like 'Request' and 'Response' in models. These are HTTP-specific. Use 'GetCatResult' or, better yet, just 'Cat' instead." + }, + { + "language": "python", + "badCode": "def create_cat(self, cat: CreateCatRequest, **kwargs) -> Cat", + "goodCode": "def create_cat(self, cat: Cat, **kwargs) -> Cat", + "comment": "Avoid using terms like 'Request' and 'Response' in models. These are HTTP-specific. Use 'Cat' instead." + } ] \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json index 79458428638..f7d2501acba 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json +++ b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json @@ -4,13 +4,12 @@ { "rule_ids": [ "python_design.html#python-client-naming", - "python_design.html#python-namespaces-async", - "python_design.html#python-client-separate-sync-async" + "python_design.html#python-namespaces-async" ], "line_no": 30, "bad_code": "class azure.contoso.NoodleAsyncManager:", - "suggestion": "Separate client classes for synchronous and asynchronous operations.", - "comment": "Service client types should be named with a Client suffix. Use an .aio suffix added to the namespace of the sync client for async clients. Do not combine async and sync operations in the same class." + "suggestion": "class azure.contoso.aio.NoodleAsyncClient:", + "comment": "The class name should end with a 'Client' suffix. The namespace should have an '.aio' suffix for async clients." }, { "rule_ids": [ @@ -19,17 +18,16 @@ "line_no": 31, "bad_code": "async def __init__(self, endpoint, credential, options: dict)", "suggestion": "async def __init__(self, endpoint, credential, **kwargs)", - "comment": "Accept optional default request options as keyword arguments and pass them along to its pipeline policies." + "comment": "The class should accept optional default request options as keyword arguments and pass them along to its pipeline policies." }, { "rule_ids": [ - "python_design.html#python-client-naming", - "python_design.html#python-client-separate-sync-async" + "python_design.html#python-client-naming" ], "line_no": 43, "bad_code": "class azure.contoso.NoodleManager:", - "suggestion": "Provide two separate client classes for synchronous and asynchronous operations.", - "comment": "The class name should have a 'Client' suffix. Do not combine async and sync operations in the same class." + "suggestion": "class azure.contoso.NoodleClient:", + "comment": "Service client types should be named with a Client suffix." }, { "rule_ids": [ @@ -37,8 +35,8 @@ ], "line_no": 49, "bad_code": "connection_string: Optional[str],", - "suggestion": "Remove the connection_string parameter from the constructor and create a separate factory classmethod from_connection_string to create a client from a connection string.", - "comment": "The constructor (__init__ method) must not take a connection string." + "suggestion": "Remove the connection_string parameter from the constructor and add a separate factory classmethod from_connection_string to create a client from a connection string.", + "comment": "The constructor should not take a connection string. Use a separate factory classmethod from_connection_string to create a client from a connection string." } ] } \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py index c5e1d27a305..b7497b8e3a5 100644 --- a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py +++ b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py @@ -6,7 +6,7 @@ from langchain.output_parsers import PydanticOutputParser import openai import re -from typing import List, Union +from typing import List, Union, Dict, Any, Optional from ._sectioned_document import SectionedDocument, Section from ._models import GuidelinesResult, Violation @@ -28,10 +28,19 @@ class GptReviewer: - def __init__(self): + + def __init__(self, log_prompt: bool = False): self.llm = AzureChatOpenAI(client=openai.ChatCompletion, deployment_name="gpt-4", openai_api_version=OPENAI_API_VERSION, temperature=0) self.output_parser = PydanticOutputParser(pydantic_object=GuidelinesResult) - + if log_prompt: + # remove the folder if it exists + base_path = os.path.join(_PACKAGE_ROOT, "scratch", "prompts") + if os.path.exists(base_path): + import shutil + shutil.rmtree(base_path) + os.makedirs(base_path) + os.environ["APIVIEW_LOG_PROMPT"] = str(log_prompt) + os.environ["APIVIEW_PROMPT_INDEX"] = "0" self.prompt_template = PromptTemplate( input_variables=["apiview", "guidelines", "language", "extra_comments", "class_list"], @@ -212,3 +221,29 @@ def retrieve_guidelines(self, language): def get_class_list(self, apiview) -> List[str]: return re.findall(r'class ([\w\.]+)', apiview) + + +# custom monkey patch to save the prompts +def _custom_generate( + self, + input_list: List[Dict[str, Any]], + run_manager: Optional["CallbackManagerForChainRun"] = None, +) -> "LLMResult": + """Generate LLM result from inputs.""" + prompts, stop = self.prep_prompts(input_list, run_manager=run_manager) + log_prompt = os.getenv("APIVIEW_LOG_PROMPT", "False").lower() == "true" + if log_prompt: + base_path = os.path.join(_PACKAGE_ROOT, "scratch", "prompts") + for prompt in prompts: + request_no = os.environ.get("APIVIEW_PROMPT_INDEX", 0) + filepath = os.path.join(base_path, f"prompt_{request_no}.txt") + with open(filepath, "w") as f: + f.write(prompt.text) + os.environ["APIVIEW_PROMPT_INDEX"] = str(int(request_no) + 1) + return self.llm.generate_prompt( + prompts, + stop, + callbacks=run_manager.get_child() if run_manager else None, + **self.llm_kwargs, + ) +LLMChain.generate = _custom_generate \ No newline at end of file From c32b8cf2678782f5425aaca6463c556e2a4518f8 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Wed, 13 Sep 2023 15:29:50 -0700 Subject: [PATCH 20/93] Tweaks. (#6969) --- .../python-packages/apiview-gpt/README.md | 36 +++++++++++++++---- packages/python-packages/apiview-gpt/cli.py | 9 ++--- .../apiview-gpt/src/_gpt_reviewer.py | 10 +++--- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/packages/python-packages/apiview-gpt/README.md b/packages/python-packages/apiview-gpt/README.md index d51d8add2a9..8534e4e0876 100644 --- a/packages/python-packages/apiview-gpt/README.md +++ b/packages/python-packages/apiview-gpt/README.md @@ -2,15 +2,22 @@ This GPT-enabled tool is designed to produce automated reviews of APIView. -## Parsing Guidelines -To generate the guidelines in JSON format: +## Getting Started + +The simplest way to get started with the project would be to follow these steps: 1. Install this package with `pip install .` or `pip install -e .` for an editable install. 2. Create a `.env` file with the following contents: ``` -AZURE_SDK_REPO_PATH="{path to the Azure SDK repo which contains the guidelines}/azure-sdk" -REST_API_GUIDELINES_PATH="{path to the REST API guidelines repo}/api-guidelines" +OPENAI_API_BASE="" # The OpenAI endpoint URL +OPENAI_API_KEY="" # The OpenAI API key +APIVIEW_GPT_SERVICE_URL=https://apiview-gpt.azurewebsites.net +APIVIEW_API_KEY="" # The APIView API key ``` -3. Run `python parse_guidelines.py`. Guidlines will be overwritten in the `guidelines` folder. +3. Create one or more test files in plain-text for the language of choice. Store them in `scratch/apiviews//`. +4. Create a `.json` file in `scratch/comments` or add to the existing one. +5. Add the documents to the vector database by running `python cli.py vector create --path `. +6. Generate a review using `python cli.py review generate --language --path `. +7. Examine the output under `scratch/output//.json`. ## Generating a Review To generate a review in JSON format: @@ -22,7 +29,11 @@ OPENAI_API_KEY="" # The OpenAI API key APIVIEW_GPT_SERVICE_URL=https://apiview-gpt.azurewebsites.net ``` 3. Run `python cli.py review generate --language --path ` -4. Output will be stored in `scratch\output\.json`. +4. Output will be stored in `scratch\output\\.json`. + +If you would like to see the raw text of the prompts sent to ChatGPT, you can append the +`--log-prompts` flag, which will dump the prompts in ascending order to a folder called `scratch/prompts`. +This folder will be wiped with each invocation and overwritten. ## Working With Semantic Documents @@ -60,6 +71,19 @@ run the following command: Here code is a text file of the APIView you want to evaluate. +If you want to see the raw output of the semantic search step, you can append the `--log-result` flag, which will dump the results +to a file called `search_result_dump.json`. This will be overwritten each time you run the command. + +## Parsing Guidelines +To generate the guidelines in JSON format: +1. Install this package with `pip install .` or `pip install -e .` for an editable install. +2. Create a `.env` file with the following contents: +``` +AZURE_SDK_REPO_PATH="{path to the Azure SDK repo which contains the guidelines}/azure-sdk" +REST_API_GUIDELINES_PATH="{path to the REST API guidelines repo}/api-guidelines" +``` +3. Run `python parse_guidelines.py`. Guidlines will be overwritten in the `guidelines` folder. + ## Documentation https://apiviewuat.azurewebsites.net/swagger/index.html diff --git a/packages/python-packages/apiview-gpt/cli.py b/packages/python-packages/apiview-gpt/cli.py index 17111897453..c4a25188fc3 100644 --- a/packages/python-packages/apiview-gpt/cli.py +++ b/packages/python-packages/apiview-gpt/cli.py @@ -35,7 +35,7 @@ def get_document(document_id: str): def create_document(path: str): """ - Create a new vector document + Add an array of vector documents to the database. """ db = VectorDB() # resolve full path @@ -72,12 +72,12 @@ def search_documents(language: str, path: str, log_result: bool = False): json.dump(results, f, indent=4) pprint(results) -def generate_review(language: str, path: str, log_prompt: bool = False): +def generate_review(language: str, path: str, log_prompts: bool = False): """ Generate a review for an APIView """ from src import GptReviewer - rg = GptReviewer(log_prompt=log_prompt) + rg = GptReviewer(log_prompts=log_prompts) filename = os.path.splitext(os.path.basename(path))[0] with open(path, "r") as f: @@ -122,11 +122,12 @@ def load_arguments(self, command): with ArgumentsContext(self, "vector") as ac: ac.argument("document_id", type=str, help="The ID of the document to retrieve", options_list=("--id")) ac.argument("log_result", action="store_true", help="Log the search results to a file called 'search_result_dump.json'") + ac.argument("path", type=str, help="The path to a JSON file containing an array of vector documents to add.") with ArgumentsContext(self, "guidelines") as ac: ac.argument("path", type, help="The path to the guidelines") with ArgumentsContext(self, "review") as ac: ac.argument("path", type=str, help="The path to the APIView file") - ac.argument("log_prompt", action="store_true", help="Log the prompt to a file called 'prompt.json'") + ac.argument("log_prompts", action="store_true", help="Log each prompt in ascending order in the `scratch/propmts` folder.") super(CliCommandsLoader, self).load_arguments(command) diff --git a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py index b7497b8e3a5..34d1c803b60 100644 --- a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py +++ b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py @@ -29,17 +29,17 @@ class GptReviewer: - def __init__(self, log_prompt: bool = False): + def __init__(self, s: bool = False): self.llm = AzureChatOpenAI(client=openai.ChatCompletion, deployment_name="gpt-4", openai_api_version=OPENAI_API_VERSION, temperature=0) self.output_parser = PydanticOutputParser(pydantic_object=GuidelinesResult) - if log_prompt: + if log_prompts: # remove the folder if it exists base_path = os.path.join(_PACKAGE_ROOT, "scratch", "prompts") if os.path.exists(base_path): import shutil shutil.rmtree(base_path) os.makedirs(base_path) - os.environ["APIVIEW_LOG_PROMPT"] = str(log_prompt) + os.environ["APIVIEW_LOG_PROMPT"] = str(log_prompts) os.environ["APIVIEW_PROMPT_INDEX"] = "0" self.prompt_template = PromptTemplate( @@ -231,8 +231,8 @@ def _custom_generate( ) -> "LLMResult": """Generate LLM result from inputs.""" prompts, stop = self.prep_prompts(input_list, run_manager=run_manager) - log_prompt = os.getenv("APIVIEW_LOG_PROMPT", "False").lower() == "true" - if log_prompt: + log_prompts = os.getenv("APIVIEW_LOG_PROMPT", "False").lower() == "true" + if log_prompts: base_path = os.path.join(_PACKAGE_ROOT, "scratch", "prompts") for prompt in prompts: request_no = os.environ.get("APIVIEW_PROMPT_INDEX", 0) From da529867d04800cb1a246ade3b161d02b566593b Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Thu, 14 Sep 2023 10:58:09 -0700 Subject: [PATCH 21/93] [APIView GPT] Streamline tokens and de-duplicate comments (#6972) * Streamline tokens. Fix bug. Deduplicate comments. * Refactor pipeline. --- .../scratch/output/python/acr_clean.json | 32 ++++++- .../scratch/output/python/noodle.json | 32 ++++--- .../apiview-gpt/src/_gpt_reviewer.py | 84 ++++++++++++------- 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json index d6c71837344..3ec4dafd9bd 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json +++ b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json @@ -1,4 +1,32 @@ { - "status": "Success", - "violations": [] + "status": "Error", + "violations": [ + { + "rule_ids": [ + "python_design.html#python-client-naming" + ], + "line_no": 21, + "bad_code": "class azure.containerregistry.ArtifactManifestProperties:", + "suggestion": "class azure.containerregistry.ArtifactManifest:", + "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'ArtifactManifest' instead." + }, + { + "rule_ids": [ + "python_design.html#python-client-naming" + ], + "line_no": 66, + "bad_code": "class azure.containerregistry.ArtifactTagProperties:", + "suggestion": "class azure.containerregistry.ArtifactTag:", + "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'ArtifactTag' instead." + }, + { + "rule_ids": [ + "python_design.html#python-client-naming" + ], + "line_no": 300, + "bad_code": "class azure.containerregistry.RepositoryProperties:", + "suggestion": "class azure.containerregistry.Repository:", + "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'Repository' instead." + } + ] } \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json index f7d2501acba..aab34426954 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json +++ b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json @@ -4,21 +4,22 @@ { "rule_ids": [ "python_design.html#python-client-naming", - "python_design.html#python-namespaces-async" + "python_design.html#python-namespaces-async", + "python_design.html#python-client-separate-sync-async" ], "line_no": 30, "bad_code": "class azure.contoso.NoodleAsyncManager:", - "suggestion": "class azure.contoso.aio.NoodleAsyncClient:", - "comment": "The class name should end with a 'Client' suffix. The namespace should have an '.aio' suffix for async clients." + "suggestion": "Separate classes for sync and async operations.", + "comment": "The class name should end with 'Client' and be in a namespace with '.aio' suffix. The class should be in a namespace with '.aio' suffix. Do not combine async and sync operations in the same class." }, { "rule_ids": [ - "python_design.html#python-client-constructor-policy-arguments" + "python_design.html#python-client-separate-sync-async" ], - "line_no": 31, - "bad_code": "async def __init__(self, endpoint, credential, options: dict)", - "suggestion": "async def __init__(self, endpoint, credential, **kwargs)", - "comment": "The class should accept optional default request options as keyword arguments and pass them along to its pipeline policies." + "line_no": 41, + "bad_code": "async def get_noodles_async(self, options: dict) -> List[NoodleResponse]", + "suggestion": "async def list_noodles_async(self, options: dict) -> List[NoodleResponse]", + "comment": "Use 'list_' prefix for a collection." }, { "rule_ids": [ @@ -27,16 +28,23 @@ "line_no": 43, "bad_code": "class azure.contoso.NoodleManager:", "suggestion": "class azure.contoso.NoodleClient:", - "comment": "Service client types should be named with a Client suffix." + "comment": "The class name should end with 'Client'." }, { "rule_ids": [ "python_design.html#python-client-connection-string" ], "line_no": 49, - "bad_code": "connection_string: Optional[str],", - "suggestion": "Remove the connection_string parameter from the constructor and add a separate factory classmethod from_connection_string to create a client from a connection string.", - "comment": "The constructor should not take a connection string. Use a separate factory classmethod from_connection_string to create a client from a connection string." + "bad_code": "connection_string: Optional[str]", + "suggestion": "Implement a separate factory method 'from_connection_string' to parse the connection string.", + "comment": "The constructor should not accept a connection string as a parameter." + }, + { + "rule_ids": [], + "line_no": 53, + "bad_code": "def create_noodle(self, body: NoodleCreateRequest, **kwargs) -> NoodleResponse", + "suggestion": "def create_noodle(self, body: Noodle, **kwargs) -> Noodle", + "comment": "Avoid using terms like 'Request' and 'Response' in models. These are HTTP-specific." } ] } \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py index 34d1c803b60..775a87e9e66 100644 --- a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py +++ b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py @@ -1,7 +1,7 @@ import os import json from langchain.chains import LLMChain -from langchain.prompts import PromptTemplate +from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate from langchain.chat_models import AzureChatOpenAI from langchain.output_parsers import PydanticOutputParser import openai @@ -29,7 +29,7 @@ class GptReviewer: - def __init__(self, s: bool = False): + def __init__(self, log_prompts: bool = False): self.llm = AzureChatOpenAI(client=openai.ChatCompletion, deployment_name="gpt-4", openai_api_version=OPENAI_API_VERSION, temperature=0) self.output_parser = PydanticOutputParser(pydantic_object=GuidelinesResult) if log_prompts: @@ -42,29 +42,27 @@ def __init__(self, s: bool = False): os.environ["APIVIEW_LOG_PROMPT"] = str(log_prompts) os.environ["APIVIEW_PROMPT_INDEX"] = "0" - self.prompt_template = PromptTemplate( - input_variables=["apiview", "guidelines", "language", "extra_comments", "class_list"], - partial_variables={"format_instructions": self.output_parser.get_format_instructions()}, - template=""" - You are trying to analyze an API for {language} to determine whether it meets the SDK guidelines. We only provide one class at a time right now, but if you need it, here's a list of all the classes in this API: - {class_list} - - Here is the code for a single class: - ``` - {apiview} - ``` + system_prompt = SystemMessagePromptTemplate.from_template(""" +You are trying to analyze an API for {language} to determine whether it meets the SDK guidelines. +We only provide one class at a time right now, but if you need it, here's a list of all the classes in this API: +{class_list} +""") + human_prompt = HumanMessagePromptTemplate.from_template(""" +Given the following guidelines: +{guidelines} - Identify any violations of the following guidelines: - {guidelines} - - Consider the following comments as well: - {extra_comments} +Evaluate the following class for any violations: +``` +{apiview} +``` + +{format_instructions} +""") + prompt_template = ChatPromptTemplate.from_messages([system_prompt, human_prompt]) + self.chain = LLMChain(llm=self.llm, prompt=prompt_template) - Format the output according to the following: - {format_instructions} - """ - ) - self.chain = LLMChain(llm=self.llm, prompt=self.prompt_template) + def _hash(self, obj) -> str: + return str(hash(json.dumps(obj))) def get_response(self, apiview, language): apiview = self.unescape(apiview) @@ -81,27 +79,54 @@ def get_response(self, apiview, language): semantic_matches = VectorDB().search_documents(language, chunk) guidelines_to_check = [] - extra_comments = [] + extra_comments = {} # extract the unique guidelines to include in the prompt grounding. # documents not included in the prompt grounding will be treated as extra comments. for match in semantic_matches: - guideline_ids = match["aiCommentModel"]["guidelineIds"] + + comment_model = match["aiCommentModel"] + if comment_model["isDeleted"] == True: + continue + + guideline_ids = comment_model["guidelineIds"] + goodCode = comment_model["goodCode"] + comment = comment_model["comment"] + if guideline_ids: guidelines_to_check.extend(guideline_ids) - else: - extra_comments.append(match["aiCommentModel"]) + + # remove unnecessary or empty fields to conserve tokens and not confuse the AI + del comment_model["id"] + del comment_model["language"] + del comment_model["embedding"] + del comment_model["guidelineIds"] + del comment_model["changeHistory"] + del comment_model["isDeleted"] + if not comment_model["goodCode"]: + del comment_model["goodCode"] + if not comment_model["comment"]: + del comment_model["comment"] + + if goodCode or comment: + extra_comments[self._hash(comment_model)] = comment_model + if not goodCode and not comment and not guideline_ids: + comment_model["comment"] = "Please have an architect look at this." + extra_comments[self._hash(comment_model)] = comment_model guidelines_to_check = list(set(guidelines_to_check)) if not guidelines_to_check: continue guidelines = self.select_guidelines(all_guidelines, guidelines_to_check) + # append the extra comments to the list of guidelines to treat them equally. + guidelines.extend(list(extra_comments.values())) + params = { "apiview": str(chunk), "guidelines": guidelines, "language": language, "class_list": class_list, - "extra_comments": extra_comments + "format_instructions": self.output_parser.get_format_instructions() } results = self.chain.run(**params) output = self.output_parser.parse(results) @@ -238,7 +263,8 @@ def _custom_generate( request_no = os.environ.get("APIVIEW_PROMPT_INDEX", 0) filepath = os.path.join(base_path, f"prompt_{request_no}.txt") with open(filepath, "w") as f: - f.write(prompt.text) + for message in prompt.messages: + f.write(message.content + "\n") os.environ["APIVIEW_PROMPT_INDEX"] = str(int(request_no) + 1) return self.llm.generate_prompt( prompts, From 4690164d26a6edd4f1ff635feeef69c705c128b4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 14 Sep 2023 18:15:07 -0400 Subject: [PATCH 22/93] [stress] Add parallel pod template to stress-test-addons (#6975) * Add parallel job template * Add parallel job example * Bump stress example dependencies * Document parallel pod usage * Add stress-test-addons 0.2.1 * Add --overwrite to stress-test-addons index deploy command * Cleanup fixes * Add comment explaining parallel argument --- tools/stress-cluster/chaos/README.md | 39 +++++++++++++++++++ .../network-stress-example/Chart.lock | 6 +-- .../network-stress-example/Chart.yaml | 2 +- .../Chart.lock | 6 +-- .../Chart.yaml | 2 +- .../examples/parallel-pod-example/.gitignore | 6 +++ .../examples/parallel-pod-example/Chart.lock | 6 +++ .../examples/parallel-pod-example/Chart.yaml | 14 +++++++ .../scenarios-matrix.yaml | 4 ++ .../stress-test-resources.bicep | 13 +++++++ .../templates/parallel-pod.yaml | 21 ++++++++++ .../stress-debug-share-example/Chart.lock | 6 +-- .../stress-debug-share-example/Chart.yaml | 2 +- .../stress-deployment-example/Chart.lock | 6 +-- .../stress-deployment-example/Chart.yaml | 2 +- .../stress-test-addons/CHANGELOG.md | 6 +++ .../kubernetes/stress-test-addons/deploy.ps1 | 2 +- .../kubernetes/stress-test-addons/index.yaml | 11 +++++- .../templates/_stress_test.tpl | 25 ++++++++++++ 19 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/.gitignore create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/stress-test-resources.bicep create mode 100644 tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml diff --git a/tools/stress-cluster/chaos/README.md b/tools/stress-cluster/chaos/README.md index dae273879a8..3cec8266d72 100644 --- a/tools/stress-cluster/chaos/README.md +++ b/tools/stress-cluster/chaos/README.md @@ -17,6 +17,7 @@ The chaos environment is an AKS cluster (Azure Kubernetes Service) with several * [Customize Docker Build](#customize-docker-build) * [Manifest Special Fields](#manifest-special-fields) * [Job Manifest](#job-manifest) + * [Run multiple pods in parallel within a test job](#run-multiple-pods-in-parallel-within-a-test-job) * [Built-In Labels](#built-in-labels) * [Chaos Manifest](#chaos-manifest) * [Scenarios and scenarios-matrix.yaml](#scenarios-and-scenarios-matrixyaml) @@ -374,6 +375,44 @@ spec: {{- end -}} ``` +#### Run multiple pods in parallel within a test job + +In some cases it may be necessary to run multiple instances of the same process/container in parallel as part of a test, +for example an eventhub test that needs to run 3 consumers, each in their own container. This can be achieved using +the `stress-test-addons.parallel-deploy-job-template.from-pod` template. The parallel feature leverages the +[job completion mode](https://kubernetes.io/docs/concepts/workloads/controllers/job/#completion-mode) feature. Test +commands in the container can read the `JOB_COMPLETION_INDEX` environment variable to make decisions. For example, +a messaging test that needs to run a single producer and multiple consumers can have logic that runs the producer when +`JOB_COMPLETION_INDEX` is 0, and a consumer when it is not 0. + +See the below example to enable parallel pods. Note the `(list . "stress.parallel-pod-example 3)` segment. The final argument (shown as `3` in the example) sets how many parallel pods should be run. + +See a full working example of parallel pods [here](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/examples/parallel-pod-example). + +``` +{{- include "stress-test-addons.parallel-deploy-job-template.from-pod" (list . "stress.parallel-pod-example" 3) -}} +{{- define "stress.parallel-pod-example" -}} +metadata: + labels: + testName: "parallel-pod-example" +spec: + containers: + - name: parallel-pod-example + image: busybox + command: ['bash', '-c'] + args: + - | + echo "Completed pod instance $JOB_COMPLETION_INDEX" + {{- include "stress-test-addons.container-env" . | nindent 6 }} +{{- end -}} +``` + +NOTE: when multiple pods are run, each pod will invoke its own azure deployment init container. When many of these containers +are run, it can cause race conditions with the arm/bicep deployment. There is logic in the deploy container to +run the full deployment in pod 0 only, and to wait on deployment completion for pods > 0. After the deployment completes, +pods > 0 start their bicep deployment, which ends up being a no-op. As a result, the main container of pod 0 will start +a little bit earlier than pods > 0. + #### Built-In Labels - `chaos` - set this to "true" to enable chaos for your pod diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock index 41b6ec1d66c..28b9fa295cd 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.0 -digest: sha256:53cbe4c0fed047f6c611523bd34181b21a310e7a3a21cb14f649bb09e4a77648 -generated: "2022-10-31T16:06:08.3743568-04:00" + version: 0.2.1 +digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 +generated: "2023-09-14T17:56:51.57311516-04:00" diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml index 9892647d648..05692aa8294 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: 0.2.0 + version: ~0.2.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock index e238fdb2fc3..94a4c075909 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.0 -digest: sha256:53cbe4c0fed047f6c611523bd34181b21a310e7a3a21cb14f649bb09e4a77648 -generated: "2022-10-31T15:58:06.8898271-04:00" + version: 0.2.1 +digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 +generated: "2023-09-14T17:56:17.165505776-04:00" diff --git a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml index 08190af7f54..7d6612c4965 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: 0.2.0 + version: ~0.2.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/.gitignore b/tools/stress-cluster/chaos/examples/parallel-pod-example/.gitignore new file mode 100644 index 00000000000..585254ec259 --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/.gitignore @@ -0,0 +1,6 @@ +# generated based on the values in your scenarios-matrix.yaml +generatedValues.yaml + +# the temp output when we compile your .bicep template with resources +stress-test-resources.json + diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock new file mode 100644 index 00000000000..127ef0fab41 --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: stress-test-addons + repository: https://stresstestcharts.blob.core.windows.net/helm/ + version: 0.2.1 +digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 +generated: "2023-09-14T17:57:06.08946796-04:00" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml new file mode 100644 index 00000000000..d8813bfb611 --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: parallel-deployment-example +description: An example stress test chart for running parallel pods within a single job +version: 0.1.1 +appVersion: v0.1 +annotations: + stressTest: 'true' # enable auto-discovery of this test via `find-all-stress-packages.ps1` + example: 'true' # enable auto-discovery filtering `find-all-stress-packages.ps1 -filters @{example='true'}` + namespace: 'examples' + +dependencies: +- name: stress-test-addons + version: ~0.2.0 + repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml new file mode 100644 index 00000000000..15eff91530f --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml @@ -0,0 +1,4 @@ +matrix: + scenarios: + parallel: + description: "Example for running multiple test containers in parallel" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/stress-test-resources.bicep b/tools/stress-cluster/chaos/examples/parallel-pod-example/stress-test-resources.bicep new file mode 100644 index 00000000000..ee8cbb40b6d --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/stress-test-resources.bicep @@ -0,0 +1,13 @@ +// Unique short string safe for naming resources like storage, service bus. +param BaseName string = '' + +resource config 'Microsoft.AppConfiguration/configurationStores@2020-07-01-preview' = { + name: 'stress-${BaseName}' + location: resourceGroup().location + sku: { + name: 'Standard' + } +} + +output RESOURCE_GROUP string = resourceGroup().name +output APP_CONFIG_NAME string = config.name diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml new file mode 100644 index 00000000000..8e51a8f36df --- /dev/null +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml @@ -0,0 +1,21 @@ +{{- /* + The 3rd argument to this template (set as `3` below) is what determines the parallel pod count. +*/}} +{{- include "stress-test-addons.parallel-deploy-job-template.from-pod" (list . "stress.parallel-pod-example" 3) -}} +{{- define "stress.parallel-pod-example" -}} +metadata: + labels: + testName: "parallel-pod-example" +spec: + containers: + - name: parallel-pod-example + image: mcr.microsoft.com/azure-cli + command: ['bash', '-c'] + args: + - | + source $ENV_FILE && + az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID && + az appconfig show -n $APP_CONFIG_NAME -g $RESOURCE_GROUP --subscription $AZURE_SUBSCRIPTION_ID -o table && + echo "Completed pod instance $JOB_COMPLETION_INDEX" + {{- include "stress-test-addons.container-env" . | nindent 6 }} +{{- end -}} diff --git a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock index f1a8a9f8edb..62aa9eaba57 100644 --- a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.0 -digest: sha256:53cbe4c0fed047f6c611523bd34181b21a310e7a3a21cb14f649bb09e4a77648 -generated: "2022-10-31T16:06:01.4558582-04:00" + version: 0.2.1 +digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 +generated: "2023-09-14T17:56:41.783433241-04:00" diff --git a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml index fd68fa3e2d5..64df7b291b4 100644 --- a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: 0.2.0 + version: ~0.2.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock index 9ef488c11fd..0afd6de2436 100644 --- a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.0 -digest: sha256:53cbe4c0fed047f6c611523bd34181b21a310e7a3a21cb14f649bb09e4a77648 -generated: "2022-10-31T15:50:26.4582611-04:00" + version: 0.2.1 +digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 +generated: "2023-09-14T17:56:01.788025676-04:00" diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml index fd3f69c6b67..3bbe19d54e8 100644 --- a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: 0.2.0 + version: ~0.2.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md index aaa556a513a..573ac51d452 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 0.2.1 (2023-09-14) + +### Features Added + +Added template for running multiple pods in parallel - `"stress-test-addons.parallel-deploy-job-template.from-pod"` + ## 0.2.0 (2022-10-21) ### Features Added diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.ps1 b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.ps1 index 4b953cebdbc..448a2085241 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.ps1 +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.ps1 @@ -30,7 +30,7 @@ RunOrExitOnFailure helm repo index --url https://stresstestcharts.blob.core.wind $files = (Get-Item *.tgz).Name $confirmation = Read-Host "Do you want to update the helm repository to add ${files}? [y/n]" if ( $confirmation -match "[yY]" ) { - RunOrExitOnFailure az storage blob upload --subscription $subscriptionId --container-name helm --file index.yaml --name index.yaml + RunOrExitOnFailure az storage blob upload --subscription $subscriptionId --container-name helm --file index.yaml --name index.yaml --overwrite RunOrExitOnFailure az storage blob upload --subscription $subscriptionId --container-name helm --file $files --name $files # index.yaml must be kept up to date, otherwise when helm generates the file, it will not diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml index 440148fe9d7..ea4f159ef06 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml @@ -1,6 +1,15 @@ apiVersion: v1 entries: stress-test-addons: + - apiVersion: v2 + appVersion: v0.1 + created: "2023-09-14T17:52:24.713925164-04:00" + description: Baseline resources and templates for stress testing clusters + digest: e8f8be8636ded8171f80aef32599fc4351d393470ef9f2143e692b0588473acb + name: stress-test-addons + urls: + - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.2.1.tgz + version: 0.2.1 - apiVersion: v2 appVersion: v0.1 created: "2022-10-21T13:08:46.8348009-07:00" @@ -172,4 +181,4 @@ entries: urls: - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.2.tgz version: 0.1.2 -generated: "2022-10-21T13:08:46.8264088-07:00" +generated: "2023-09-14T17:52:24.706686944-04:00" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl index 07207f950b9..e058426b028 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl @@ -4,6 +4,15 @@ spec: {{- include (index . 1) (index . 0) | nindent 4 -}} {{- end -}} +{{- define "stress-test-addons.parallel-job-wrapper.tpl" -}} +spec: + completions: {{ index . 2 }} + parallelism: {{ index . 2 }} + completionMode: Indexed + template: + {{- include (index . 1) (index . 0) | nindent 4 -}} +{{- end -}} + {{- define "stress-test-addons.deploy-job-template.tpl" -}} apiVersion: batch/v1 kind: Job @@ -58,6 +67,22 @@ spec: {{- include "stress-test-addons.static-secrets" $global }} {{- end -}} +{{- define "stress-test-addons.parallel-deploy-job-template.from-pod" -}} +{{- $global := index . 0 -}} +{{- $podDefinition := index . 1 -}} +{{- $parallel := index . 2 -}} +# Configmap template that adds the stress test ARM template for mounting +{{- include "stress-test-addons.deploy-configmap" $global }} +{{- range (default (list "stress") $global.Values.scenarios) }} +--- +{{ $jobCtx := fromYaml (include "stress-test-addons.util.mergeStressContext" (list $global . )) }} +{{- $jobOverride := fromYaml (include "stress-test-addons.parallel-job-wrapper.tpl" (list $jobCtx $podDefinition $parallel)) -}} +{{- $tpl := fromYaml (include "stress-test-addons.deploy-job-template.tpl" $jobCtx) -}} +{{- toYaml (merge $jobOverride $tpl) -}} +{{- end }} +{{- include "stress-test-addons.static-secrets" $global }} +{{- end -}} + {{- define "stress-test-addons.env-job-template.tpl" -}} apiVersion: batch/v1 kind: Job From ce6578cf7f72970fd5d0e655b5bf37c0d9b85e5c Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Fri, 15 Sep 2023 08:36:30 -0700 Subject: [PATCH 23/93] fix setup (#6978) --- packages/python-packages/apiview-gpt/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-packages/apiview-gpt/setup.py b/packages/python-packages/apiview-gpt/setup.py index 549952b5c89..e1d96a1a164 100644 --- a/packages/python-packages/apiview-gpt/setup.py +++ b/packages/python-packages/apiview-gpt/setup.py @@ -7,7 +7,7 @@ "A tool for generating APIView reviews using GPT-4." ) -with open(os.path.join("apiview-gpt", "_version.py"), "r") as fd: +with open(os.path.join("src", "_version.py"), "r") as fd: version = re.search( r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE ).group(1) From df198a9802944754d5ff8e3927dbddc1b9e87fd6 Mon Sep 17 00:00:00 2001 From: KarishmaGhiya Date: Fri, 15 Sep 2023 08:41:11 -0700 Subject: [PATCH 24/93] update typescript api view and comment samples (#6976) --- .../scratch/apiviews/typescript/openai.txt | 179 ++++++++++++++++++ .../scratch/apiviews/typescript/pasta.txt | 48 +++++ .../scratch/comments/typescript.json | 38 ++++ 3 files changed, 265 insertions(+) create mode 100644 packages/python-packages/apiview-gpt/scratch/apiviews/typescript/openai.txt create mode 100644 packages/python-packages/apiview-gpt/scratch/apiviews/typescript/pasta.txt create mode 100644 packages/python-packages/apiview-gpt/scratch/comments/typescript.json diff --git a/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/openai.txt b/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/openai.txt new file mode 100644 index 00000000000..0634718a865 --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/openai.txt @@ -0,0 +1,179 @@ +export interface AzureChatExtensionConfiguration { + parameters: Record; + type: AzureChatExtensionType; +} +export interface AzureChatExtensionsMessageContext { + messages?: ChatMessage[]; +} +export type AzureChatExtensionType = string; +export interface AzureExtensionsOptions { + extensions?: AzureChatExtensionConfiguration[]; +} +export type AzureOpenAIOperationState = string; +export interface BatchImageGenerationOperationResponse { + created: Date; + error?: ErrorModel; + expires?: number; + id: string; + result?: ImageGenerations; + status: AzureOpenAIOperationState; +} +export interface ChatChoice { + contentFilterResults?: ContentFilterResults; + delta?: ChatMessage; + finishReason: CompletionsFinishReason | null; + index: number; + message?: ChatMessage; +} +export interface ChatCompletions { + choices: ChatChoice[]; + created: Date; + id: string; + promptFilterResults?: PromptFilterResult[]; + usage?: CompletionsUsage; +} +export interface ChatMessage { + content: string | null; + context?: AzureChatExtensionsMessageContext; + functionCall?: FunctionCall; + name?: string; + role: ChatRole; +} +export type ChatRole = string; +export interface Choice { + contentFilterResults?: ContentFilterResults; + finishReason: CompletionsFinishReason | null; + index: number; + logprobs: CompletionsLogProbabilityModel | null; + text: string; +} +export interface Completions { + choices: Choice[]; + created: Date; + id: string; + promptFilterResults?: PromptFilterResult[]; + usage: CompletionsUsage; +} +export type CompletionsFinishReason = string; +export interface CompletionsLogProbabilityModel { + textOffset: number[]; + tokenLogprobs: (number | null)[]; + tokens: string[]; + topLogprobs: Record[]; +} +export interface CompletionsUsage { + completionTokens: number; + promptTokens: number; + totalTokens: number; +} +export interface ContentFilterResult { + filtered: boolean; + severity: ContentFilterSeverity; +} +export interface ContentFilterResults { + hate?: ContentFilterResult; + selfHarm?: ContentFilterResult; + sexual?: ContentFilterResult; + violence?: ContentFilterResult; +} +export type ContentFilterSeverity = string; +export interface EmbeddingItem { + embedding: number[]; + index: number; +} +export interface Embeddings { + data: EmbeddingItem[]; + usage: EmbeddingsUsage; +} +export interface EmbeddingsUsage { + promptTokens: number; + totalTokens: number; +} +export interface FunctionCall { + arguments: string; + name: string; +} +export type FunctionCallPreset = string; +export interface FunctionDefinition { + description?: string; + name: string; + parameters?: Record; +} +export interface FunctionName { + name: string; +} +export interface GetChatCompletionsOptions extends OperationOptions { + azureExtensionOptions?: AzureExtensionsOptions; + frequencyPenalty?: number; + functionCall?: FunctionCallPreset | FunctionName; + functions?: FunctionDefinition[]; + logitBias?: Record; + maxTokens?: number; + model?: string; + n?: number; + presencePenalty?: number; + stop?: string[]; + stream?: boolean; + temperature?: number; + topP?: number; + user?: string; +} +export interface GetCompletionsOptions extends OperationOptions { + bestOf?: number; + echo?: boolean; + frequencyPenalty?: number; + logitBias?: Record; + logprobs?: number; + maxTokens?: number; + model?: string; + n?: number; + presencePenalty?: number; + stop?: string[]; + stream?: boolean; + temperature?: number; + topP?: number; + user?: string; +} +export interface GetEmbeddingsOptions extends OperationOptions { + model?: string; + user?: string; +} +export interface ImageGenerationOptions extends OperationOptions { + n?: number; + responseFormat?: ImageGenerationResponseFormat; + size?: ImageSize; + user?: string; +} +export type ImageGenerationResponseFormat = string; +export interface ImageGenerations { + created: Date; + data: ImageLocation[] | ImagePayload[]; +} +export interface ImageLocation { + url: string; +} +export interface ImagePayload { + base64Data: string; +} +export type ImageSize = string; +export declare class OpenAIClient { + constructor(endpoint: string, credential: KeyCredential, options?: OpenAIClientOptions); + constructor(endpoint: string, credential: TokenCredential, options?: OpenAIClientOptions); + constructor(openAiApiKey: KeyCredential, options?: OpenAIClientOptions); + getChatCompletions(deploymentName: string, messages: ChatMessage[], options?: GetChatCompletionsOptions): Promise; + getCompletions(deploymentName: string, prompt: string[], options?: GetCompletionsOptions): Promise; + getEmbeddings(deploymentName: string, input: string[], options?: GetEmbeddingsOptions): Promise; + getImages(prompt: string, options?: ImageGenerationOptions): Promise; + listChatCompletions(deploymentName: string, messages: ChatMessage[], options?: GetChatCompletionsOptions): AsyncIterable; + listCompletions(deploymentName: string, prompt: string[], options?: GetCompletionsOptions): AsyncIterable>; +} +export interface OpenAIClientOptions extends ClientOptions { } +export declare class OpenAIKeyCredential implements KeyCredential { + constructor(key: string); + get key(): string; + update(newKey: string): void; +} +export interface PromptFilterResult { + contentFilterResults?: ContentFilterResults; + promptIndex: number; +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/pasta.txt b/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/pasta.txt new file mode 100644 index 00000000000..52ae85222ea --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/apiviews/typescript/pasta.txt @@ -0,0 +1,48 @@ +export interface pastaType { + lengthInCms: number; + widthInCms?: number; + flour: string; + NoodleType: string; + containsEgg?: boolean; +} + +export class Pasta { + name: string; + sauceType: SauceType; + pastaType: pastaType; + ingredients: string[]; + cheeseTypes: string[]; + glutenFree?: boolean; + vegan?: boolean; + constructor(name: string); + constructor(name: string, sauceType?: SauceType, pastaType?: pastaType, ingredients?: string[], cheeseTypes?: string[]); + constructor(name: string, sauceType: SauceType, pastaType: pastaType, ingredients: string[], cheeseTypes: string[]); +} + +export type SauceType: "alfredo" | "marinara" | "bolognese" | "amatriciana" | "arrabbiata" | "carbonara" | "pesto" + +export class PastaLaVista { + endpoint?: string; + tokenCredential: TokenCredential; + options?: CommonClientOptions; + function retrieveAllPastas(options?: pastaOptions): PagedAsyncIterableIterator; +} + +export interface pastaOptions { + pastaType?: pastaType; + sauceType?: sauceType; +} + +export type PastaListResponse = WithResponse; + +export interface PastaListHeaders { + contentType?: string; + date?: Date; + errorCode?: string; + requestId?: string; + version?: string; +} +export interface ListPastasResponse { + continuationToken: string; + pastaList?: Pasta[]; +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/comments/typescript.json b/packages/python-packages/apiview-gpt/scratch/comments/typescript.json new file mode 100644 index 00000000000..91a467d589f --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/comments/typescript.json @@ -0,0 +1,38 @@ +[ + { + "language": "typescript", + "badCode": "class ExampleClient { \n constructor (connectionString: string, options: ExampleClientOptions); \n constructor (url: string, options: ExampleClientOptions); \n constructor (urlOrCS: string, options: ExampleClientOptions) { \n} \n}", + "goodCode": "class ExampleClient { \n constructor (url: string, options: ExampleClientOptions) { \n } \n static fromConnectionString(connectionString: string, options: ExampleClientOptions) { \n} \n}", + "guidelineIds": ["typescript_design.html#ts-use-overloads-over-unions"] + }, + { + "language": "typescript", + "badCode": "function getItems(name: string, options?: itemsOptions): Promise;\n interface itemsOptions { extensions?: configuration[]; }", + "goodCode": "function getItems(name: string, options?: getItemsOptions): Promise;\n interface getItemsOptions { extensions?: configuration[]; }", + "guidelineIds": ["typescript_design.html#ts-naming-options"] + }, + { + "language": "typescript", + "badCode": "containerClient.deleteContainer();", + "goodCode": "containerClient.delete();", + "guidelineIds": ["typescript_design.html#ts-approved-verbs"] + }, + { + "language": "typescript", + "badCode": "containerClient.createOrUpdate();", + "goodCode": "containerClient.upsert();", + "guidelineIds": ["typescript_design.html#ts-approved-verbs"] + }, + { + "language": "typescript", + "badCode": "function listItems() {\n return {\n nextItem() { /*...*/ } \n }\n }", + "goodCode": "async function* listItems() {\n for (const item of items) { \n yield item;\n }\n }", + "guidelineIds": ["typescript_design.html#ts-use-async-functions"] + }, + { + "language": "typescript", + "badCode": "async function getItems(): PagedAsyncIterableIterator{}", + "goodCode": "async function listItems(): PagedAsyncIterableIterator{}", + "guidelineIds": ["typescript_design.html#ts-pagination-provide-list"] + } +] \ No newline at end of file From 62e1769b038a13c2edc843444275b2dc817bfacf Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Fri, 15 Sep 2023 08:43:55 -0700 Subject: [PATCH 25/93] Fix ID hallucinations (#6973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix for wonky IDs. * Update ci config. --- packages/python-packages/apiview-gpt/ci.yml | 18 +++++++++++++ .../scratch/output/python/acr_clean.json | 12 +++------ .../scratch/output/python/noodle.json | 25 ++++++++++--------- .../apiview-gpt/src/_gpt_reviewer.py | 2 +- 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/packages/python-packages/apiview-gpt/ci.yml b/packages/python-packages/apiview-gpt/ci.yml index 481369c16ff..f91247be4a2 100644 --- a/packages/python-packages/apiview-gpt/ci.yml +++ b/packages/python-packages/apiview-gpt/ci.yml @@ -3,6 +3,20 @@ trigger: branches: include: - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - packages/python-packages/apiview-gpt + +pr: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* paths: include: - packages/python-packages/apiview-gpt @@ -15,3 +29,7 @@ extends: FeedName: 'public/azure-sdk-for-python' ArtifactName: 'apiviewcopilot' PackageName: 'Python API View Copilot' + TestSteps: + - script: | + echo "Pass Check Enforcer!" + displayName: 'Pass Check Enforcer' \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json index 3ec4dafd9bd..84ae202abb9 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json +++ b/packages/python-packages/apiview-gpt/scratch/output/python/acr_clean.json @@ -2,27 +2,21 @@ "status": "Error", "violations": [ { - "rule_ids": [ - "python_design.html#python-client-naming" - ], + "rule_ids": [], "line_no": 21, "bad_code": "class azure.containerregistry.ArtifactManifestProperties:", "suggestion": "class azure.containerregistry.ArtifactManifest:", "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'ArtifactManifest' instead." }, { - "rule_ids": [ - "python_design.html#python-client-naming" - ], + "rule_ids": [], "line_no": 66, "bad_code": "class azure.containerregistry.ArtifactTagProperties:", "suggestion": "class azure.containerregistry.ArtifactTag:", "comment": "The suffix 'Properties' is not helpful. Favor the simpler term 'ArtifactTag' instead." }, { - "rule_ids": [ - "python_design.html#python-client-naming" - ], + "rule_ids": [], "line_no": 300, "bad_code": "class azure.containerregistry.RepositoryProperties:", "suggestion": "class azure.containerregistry.Repository:", diff --git a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json index aab34426954..94678cd9fa4 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json +++ b/packages/python-packages/apiview-gpt/scratch/output/python/noodle.json @@ -9,26 +9,27 @@ ], "line_no": 30, "bad_code": "class azure.contoso.NoodleAsyncManager:", - "suggestion": "Separate classes for sync and async operations.", - "comment": "The class name should end with 'Client' and be in a namespace with '.aio' suffix. The class should be in a namespace with '.aio' suffix. Do not combine async and sync operations in the same class." + "suggestion": "class azure.contoso.aio.NoodleAsyncClient:", + "comment": "The class name should end with 'Client' and be in a namespace with '.aio' suffix. The class should be in a namespace with '.aio' suffix. There should be two separate classes for sync and async operations." }, { "rule_ids": [ - "python_design.html#python-client-separate-sync-async" + "python_design.html#python-client-constructor-policy-arguments" ], - "line_no": 41, - "bad_code": "async def get_noodles_async(self, options: dict) -> List[NoodleResponse]", - "suggestion": "async def list_noodles_async(self, options: dict) -> List[NoodleResponse]", - "comment": "Use 'list_' prefix for a collection." + "line_no": 31, + "bad_code": "async def __init__(self, endpoint, credential, options: dict)", + "suggestion": "async def __init__(self, endpoint, credential, **kwargs)", + "comment": "The constructor should accept optional default request options as keyword arguments." }, { "rule_ids": [ - "python_design.html#python-client-naming" + "python_design.html#python-client-naming", + "python_design.html#python-client-separate-sync-async" ], "line_no": 43, "bad_code": "class azure.contoso.NoodleManager:", - "suggestion": "class azure.contoso.NoodleClient:", - "comment": "The class name should end with 'Client'." + "suggestion": "Provide two separate client classes for synchronous and asynchronous operations.", + "comment": "The class name should end with 'Client'. The class should not combine async and sync operations." }, { "rule_ids": [ @@ -36,8 +37,8 @@ ], "line_no": 49, "bad_code": "connection_string: Optional[str]", - "suggestion": "Implement a separate factory method 'from_connection_string' to parse the connection string.", - "comment": "The constructor should not accept a connection string as a parameter." + "suggestion": "Implement a separate factory classmethod from_connection_string to create a client from a connection string.", + "comment": "The constructor should not accept a connection string directly." }, { "rule_ids": [], diff --git a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py index 775a87e9e66..908223b0f51 100644 --- a/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py +++ b/packages/python-packages/apiview-gpt/src/_gpt_reviewer.py @@ -97,7 +97,6 @@ def get_response(self, apiview, language): guidelines_to_check.extend(guideline_ids) # remove unnecessary or empty fields to conserve tokens and not confuse the AI - del comment_model["id"] del comment_model["language"] del comment_model["embedding"] del comment_model["guidelineIds"] @@ -264,6 +263,7 @@ def _custom_generate( filepath = os.path.join(base_path, f"prompt_{request_no}.txt") with open(filepath, "w") as f: for message in prompt.messages: + f.write(f"==={message.type.upper()}===\n") f.write(message.content + "\n") os.environ["APIVIEW_PROMPT_INDEX"] = str(int(request_no) + 1) return self.llm.generate_prompt( From 3e31b57cef589538eda7c43a0a57c0dafdf53eff Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Fri, 15 Sep 2023 09:30:39 -0700 Subject: [PATCH 26/93] [APIView GPT] Typescript fixes (#6980) * Generate pasta.txt output * Update split logic to work for TypeScript (and other languages that use curly braces) --- .../scratch/output/typescript/pasta.json | 33 +++++++++++++++++++ .../apiview-gpt/src/_sectioned_document.py | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json diff --git a/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json b/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json new file mode 100644 index 00000000000..ff6ac25b2b9 --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json @@ -0,0 +1,33 @@ +{ + "status": "Error", + "violations": [ + { + "rule_ids": [ + "typescript_design.html#ts-use-overloads-over-unions" + ], + "line_no": 17, + "bad_code": "constructor(name: string, sauceType?: SauceType, pastaType?: pastaType, ingredients?: string[], cheeseTypes?: string[]);", + "suggestion": "constructor(name: string);\nconstructor(name: string, sauceType: SauceType, pastaType: pastaType, ingredients: string[], cheeseTypes: string[]);", + "comment": "The constructor of the Pasta class should be overloaded properly to handle multiple correlated parameters." + }, + { + "rule_ids": [ + "typescript_design.html#ts-naming-options", + "typescript_design.html#ts-approved-verbs" + ], + "line_no": 27, + "bad_code": "function retrieveAllPastas(options?: pastaOptions): PagedAsyncIterableIterator;", + "suggestion": "function listAllPastas(options?: listAllPastasOptions): PagedAsyncIterableIterator;", + "comment": "The options parameter in the method should be named as Options. The method name should start with 'list' when it returns a PagedAsyncIterableIterator." + }, + { + "rule_ids": [ + "typescript_design.html#ts-naming-options" + ], + "line_no": 30, + "bad_code": "export interface pastaOptions {\n pastaType?: pastaType;\n sauceType?: sauceType;\n}", + "suggestion": "export interface PastaOptions {\n pastaType?: pastaType;\n sauceType?: sauceType;\n}", + "comment": "The type of the options bag should be named as Options. In this case, it should be 'PastaOptions' instead of 'pastaOptions'." + } + ] +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/src/_sectioned_document.py b/packages/python-packages/apiview-gpt/src/_sectioned_document.py index 54dacf50429..9476645c41e 100644 --- a/packages/python-packages/apiview-gpt/src/_sectioned_document.py +++ b/packages/python-packages/apiview-gpt/src/_sectioned_document.py @@ -27,7 +27,7 @@ def __init__(self, lines: List[str], chunk: bool): indent = len(line) - len(line.lstrip()) line_data.append(LineData(i, indent, line)) - top_level_lines = [x for x in line_data if x.indent == 0 and x.line != ""] + top_level_lines = [x for x in line_data if x.indent == 0 and x.line != "" and x.line != "}"] for i in range(len(top_level_lines)): line1 = top_level_lines[i] try: From 8b979fa4090dbce9d8c827c5b073df1ec56b1db7 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Fri, 15 Sep 2023 13:47:31 -0700 Subject: [PATCH 27/93] update proxy version (#6982) --- eng/common/testproxy/target_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index eaeb3436b8d..49c8aea654f 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20230818.1 +1.0.0-dev.20230912.4 From f6002502db55240b7d2fd78e7c23e7a0d876930a Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Fri, 15 Sep 2023 13:51:46 -0700 Subject: [PATCH 28/93] [APIView GPT] TypeScript document additions (#6983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Generate pasta.txt output * Update split logic to work for TypeScript (and other languages that use curly braces) * Updates for TS. --- .../scratch/comments/typescript.json | 70 ++++++++++++++++++- .../scratch/output/typescript/openai.json | 23 ++++++ .../scratch/output/typescript/pasta.json | 38 ++++++++-- 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 packages/python-packages/apiview-gpt/scratch/output/typescript/openai.json diff --git a/packages/python-packages/apiview-gpt/scratch/comments/typescript.json b/packages/python-packages/apiview-gpt/scratch/comments/typescript.json index 91a467d589f..07b26c217da 100644 --- a/packages/python-packages/apiview-gpt/scratch/comments/typescript.json +++ b/packages/python-packages/apiview-gpt/scratch/comments/typescript.json @@ -15,7 +15,7 @@ "language": "typescript", "badCode": "containerClient.deleteContainer();", "goodCode": "containerClient.delete();", - "guidelineIds": ["typescript_design.html#ts-approved-verbs"] + "guidelineIds": ["typescript_design.html#ts-approved-verbs", "typescript_design.html#ts-naming-drop-noun"] }, { "language": "typescript", @@ -25,14 +25,80 @@ }, { "language": "typescript", - "badCode": "function listItems() {\n return {\n nextItem() { /*...*/ } \n }\n }", + "badCode": "function listItems() {\n return Rx.Observable.of(/* ... */)\n }", + "goodCode": "async function* listItems() {\n for (const item of items) { \n yield item;\n }\n }", + "guidelineIds": ["typescript_design.html#ts-use-async-functions"] + }, + { + "language": "typescript", + "badCode": "function listItems() {\n // fetch items\n for (const item of items) {\n callback (item);\n }\n}", "goodCode": "async function* listItems() {\n for (const item of items) { \n yield item;\n }\n }", "guidelineIds": ["typescript_design.html#ts-use-async-functions"] }, + { + "language": "typescript", + "badCode": "function listItems() {\n return {\n nextItem() { /*...*/ } \n }\n }", + "goodCode": "function* listItems() {\n /* .... */\n}", + "comment": "Prefer a generator function over an iterator object." + }, { "language": "typescript", "badCode": "async function getItems(): PagedAsyncIterableIterator{}", "goodCode": "async function listItems(): PagedAsyncIterableIterator{}", "guidelineIds": ["typescript_design.html#ts-pagination-provide-list"] + }, + { + "language": "typescript", + "badCode": "export class ServiceClient {\n // client constructors have overloads for handling different\n // authentication schemes.\n constructor(connectionString: string, options?: ServiceClientOptions);\n constructor(host: string, credential: TokenCredential, options?: ServiceClientOptions);\n constructor(...) { }\n\n // Service methods. Options take at least an abortSignal.\n async createItem(options?: CreateItemOptions): CreateItemResponse;\n async deleteItem(options?: DeleteItemOptions): DeleteItemResponse;\n // Simple paginated API\n listItems(): PagedAsyncIterableIterator { }\n\n // Clients for sub-resources\n getItemClient(itemName: string) { }\n}", + "comment": "This is a service client class because it has a variety of methods that call APIs. The class name should end in 'Client'.", + "guidelineIds": ["typescript_design.html#ts-apisurface-serviceclientnaming"] + }, + { + "language":"typescript", + "badCode": "interface EventModel {\n /* ... */\n}", + "goodCode": "interface Event {\n /* ... */\n}", + "comment": "If an interface represents an entity transferred to and from an Azure service and can be round-tripped, it should use the simplest name that corresponds to the entity, without any extraneous suffixes.", + "guidelineIds": ["typescript_design.html#ts-model-types-use-good-name"] + }, + { + "language": "typescript", + "badCode": "interface ConfigurationSetting {\n key: string;\n value: string;\n lastModifiedOn: Date; // less important and less frequently used\n receivedOn: Date; // less important and less frequently used\n etag: string; // less important and less frequently used\n}", + "goodCode": "interface ConfigurationSettingDetails {\n lastModifiedOn: Date;\n receivedOn: Date;\n etag: string;\n}\n\ninterface ConfigurationSetting {\n key: string;\n value: string;\n details: ConfigurationSettingDetails;\n}", + "guidelineIds": ["typescript_design.html#ts-model-types-use-details"] + }, + { + "language": "typescript", + "badCode": "Foo", + "comment": "Only use if this represents the full data of the resource. It should be able to be used in create, update, and get operations.", + "guidelineIds": ["typescript_design.html#ts-model-types-partial-naming"] + }, + { + "language": "typescript", + "badCode": "FooDetails", + "comment": "Use for less important details about a resource. Must be attached to 'Foo.details'", + "guidelineIds": ["typescript_design.html#ts-model-types-partial-naming"] + }, + { + "language": "typescript", + "badCode": "FooItem", + "comment": "A single instance of 'Foo' when returned from an enumeration.", + "guidelineIds": ["typescript_design.html#ts-model-types-partial-naming"] + }, + { + "language": "typescript", + "badCode": "OperationFooOptions", + "comment": "Optional parameters that apply only to a single operation.", + "guidelineIds": ["typescript_design.html#ts-model-types-partial-naming"] + }, + { + "language": "typescript", + "badCode": "OperationFooResult", + "comment": "Used when the operation returns a partial or different set of data for a single operation. If it returns all of the properties of 'Foo', then just use 'Foo'.", + "guidelineIds": ["typescript_design.html#ts-model-types-partial-naming"] + }, + { + "language": "typescript", + "badCode": "interface ConfigurationSetting {\n key: string;\n value: string;\n details: ConfigurationSettingDetails;\n}", + "comment": "This is model data, which we represent in TypeScript as an interface. It is not a client class and should not follow client conventions." } ] \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/typescript/openai.json b/packages/python-packages/apiview-gpt/scratch/output/typescript/openai.json new file mode 100644 index 00000000000..e00362b146c --- /dev/null +++ b/packages/python-packages/apiview-gpt/scratch/output/typescript/openai.json @@ -0,0 +1,23 @@ +{ + "status": "Error", + "violations": [ + { + "rule_ids": [ + "typescript_design.html#ts-model-types-use-good-name" + ], + "line_no": 151, + "bad_code": "export interface ImageLocation {\n url: string;\n}", + "suggestion": "The interface should represent a complete entity that can be round-tripped to the service. If 'ImageLocation' is not a complete entity, consider renaming it or adding more properties to make it complete.", + "comment": "The interface 'ImageLocation' does not seem to represent a complete entity that can be round-tripped to the service." + }, + { + "rule_ids": [ + "typescript_design.html#ts-use-overloads-over-unions" + ], + "line_no": 159, + "bad_code": "constructor(endpoint: string, credential: KeyCredential, options?: OpenAIClientOptions);\nconstructor(endpoint: string, credential: TokenCredential, options?: OpenAIClientOptions);\nconstructor(openAiApiKey: KeyCredential, options?: OpenAIClientOptions);", + "suggestion": "constructor(endpoint: string, credential: KeyCredential | TokenCredential, options?: OpenAIClientOptions);\nstatic fromApiKey(openAiApiKey: KeyCredential, options?: OpenAIClientOptions);", + "comment": "The constructors are overloaded with different types of parameters which can lead to confusion and potential misuse. It's better to use union types for the `credential` parameter and provide a static method for the API key constructor." + } + ] +} \ No newline at end of file diff --git a/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json b/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json index ff6ac25b2b9..831aabd7b66 100644 --- a/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json +++ b/packages/python-packages/apiview-gpt/scratch/output/typescript/pasta.json @@ -1,6 +1,15 @@ { "status": "Error", "violations": [ + { + "rule_ids": [ + "typescript_design.html#ts-apisurface-serviceclientnaming" + ], + "line_no": 8, + "bad_code": "export class Pasta { ... }", + "suggestion": "export class PastaClient { ... }", + "comment": "The class name should end with 'Client' suffix." + }, { "rule_ids": [ "typescript_design.html#ts-use-overloads-over-unions" @@ -8,17 +17,34 @@ "line_no": 17, "bad_code": "constructor(name: string, sauceType?: SauceType, pastaType?: pastaType, ingredients?: string[], cheeseTypes?: string[]);", "suggestion": "constructor(name: string);\nconstructor(name: string, sauceType: SauceType, pastaType: pastaType, ingredients: string[], cheeseTypes: string[]);", - "comment": "The constructor of the Pasta class should be overloaded properly to handle multiple correlated parameters." + "comment": "Overloads should be preferred over unions." + }, + { + "rule_ids": [ + "typescript_design.html#ts-apisurface-serviceclientnaming" + ], + "line_no": 23, + "bad_code": "export class PastaLaVista {", + "suggestion": "export class PastaLaVistaClient {", + "comment": "The class name should end with 'Client'." }, { "rule_ids": [ - "typescript_design.html#ts-naming-options", - "typescript_design.html#ts-approved-verbs" + "typescript_design.html#ts-naming-options" ], "line_no": 27, "bad_code": "function retrieveAllPastas(options?: pastaOptions): PagedAsyncIterableIterator;", - "suggestion": "function listAllPastas(options?: listAllPastasOptions): PagedAsyncIterableIterator;", - "comment": "The options parameter in the method should be named as Options. The method name should start with 'list' when it returns a PagedAsyncIterableIterator." + "suggestion": "function retrieveAllPastas(options?: RetrieveAllPastasOptions): PagedAsyncIterableIterator;", + "comment": "The options parameter should be named as Options." + }, + { + "rule_ids": [ + "typescript_design.html#ts-use-overloads-over-unions" + ], + "line_no": 24, + "bad_code": "endpoint?: string;", + "suggestion": "Prefer using overloads over unions for correlated parameters.", + "comment": "The class does not use overloads over unions." }, { "rule_ids": [ @@ -27,7 +53,7 @@ "line_no": 30, "bad_code": "export interface pastaOptions {\n pastaType?: pastaType;\n sauceType?: sauceType;\n}", "suggestion": "export interface PastaOptions {\n pastaType?: pastaType;\n sauceType?: sauceType;\n}", - "comment": "The type of the options bag should be named as Options. In this case, it should be 'PastaOptions' instead of 'pastaOptions'." + "comment": "The options interface should be named as Options. In this case, it should be 'PastaOptions'." } ] } \ No newline at end of file From 296746f8a16779131353abfddfdd264b291a0f1b Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Fri, 15 Sep 2023 14:46:52 -0700 Subject: [PATCH 29/93] Gpt build zip (#6981) * build zip * update * Update ci.yml * Update packages/python-packages/apiview-gpt/ci.yml --- packages/python-packages/apiview-gpt/ci.yml | 29 ++++++++------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/packages/python-packages/apiview-gpt/ci.yml b/packages/python-packages/apiview-gpt/ci.yml index f91247be4a2..ed8ab416382 100644 --- a/packages/python-packages/apiview-gpt/ci.yml +++ b/packages/python-packages/apiview-gpt/ci.yml @@ -3,9 +3,6 @@ trigger: branches: include: - main - - feature/* - - release/* - - hotfix/* paths: include: - packages/python-packages/apiview-gpt @@ -14,22 +11,18 @@ pr: branches: include: - main - - feature/* - - release/* - - hotfix/* paths: include: - packages/python-packages/apiview-gpt -extends: - template: /eng/pipelines/templates/stages/archetype-sdk-tool-python.yml - parameters: - PythonVersion: '3.10' - PackagePath: 'packages/python-packages/apiview-gpt' - FeedName: 'public/azure-sdk-for-python' - ArtifactName: 'apiviewcopilot' - PackageName: 'Python API View Copilot' - TestSteps: - - script: | - echo "Pass Check Enforcer!" - displayName: 'Pass Check Enforcer' \ No newline at end of file +steps: + - script: | + zip -r app.zip . + workingDirectory: packages/python-packages/apiview-gpt + displayName: 'Package app into a zip file' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: 'packages/python-packages/apiview-gpt/app.zip' + artifactName: 'drop' + publishLocation: 'Container' From 14bf1d83f837ead4247dbca18c40cce817b1e14b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 18 Sep 2023 11:19:30 -0700 Subject: [PATCH 30/93] Serilaize with depth (#6985) --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 94aa8c1efe1..9b665dbc98d 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -205,7 +205,7 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { Write-Host "The docs metadata json $packageMetadataName does not exist, creating a new one to docs repo..." New-Item -ItemType Directory -Path $packageInfoLocation -Force } - $packageInfoJson = ConvertTo-Json $packageInfo + $packageInfoJson = ConvertTo-Json $packageInfo -Depth 100 Set-Content ` -Path $packageInfoLocation/$packageMetadataName ` -Value $packageInfoJson From 57ac0b1fea4149e4e4c9eb06d77959bed4c40f8d Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 19 Sep 2023 07:58:01 -0700 Subject: [PATCH 31/93] Fix internal links in `github-event-processor` (#6988) * update link paths to deal with the fact that README was moved at some point in the past * Apply suggestions from code review Remove extra newlines --- tools/github-event-processor/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/github-event-processor/README.md b/tools/github-event-processor/README.md index 0dfd342e05f..ee15a3a46ce 100644 --- a/tools/github-event-processor/README.md +++ b/tools/github-event-processor/README.md @@ -45,11 +45,11 @@ github-event-processor ${{ github.event_name }} payload.json **payload.json** is the toJson of the github.event redirected into file. The action that triggered the event is part of this payload. -**TaskToRun** is specific to Scheduled event processing and defines what rule to run. This string matches the rule name constant defined in the [RulesConstants](./Constants/RulesConstants.cs) file. The reason this was done this way is that it prevents the code from needing knowledge of which cron schedule string belongs to which rule. +**TaskToRun** is specific to Scheduled event processing and defines what rule to run. This string matches the rule name constant defined in the [RulesConstants](./Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs) file. The reason this was done this way is that it prevents the code from needing knowledge of which cron schedule string belongs to which rule. ### Rules Configuration -The [rules configuration file](../yml-files/event-processor.config) is simply a Json file which defines which rules are active for the repository and they're loaded up every time the GitHubEventProcessor runs. The full set rules is in the [RulesConstants](./Constants/RulesConstants.cs) file and their state is either **On** or **Off**. *Note: AzureSdk language repositories should have all rules enabled but non-language repositories, like azure-sdk-tools, have a reduced set of rules. For example: +The [rules configuration file](./YmlAndConfigFiles/event-processor.config) is simply a Json file which defines which rules are active for the repository and they're loaded up every time the GitHubEventProcessor runs. The full set rules is in the [RulesConstants](./Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs) file and their state is either **On** or **Off**. *Note: AzureSdk language repositories should have all rules enabled but non-language repositories, like azure-sdk-tools, have a reduced set of rules. For example: ```json "InitialIssueTriage": "On", From 437d8be5636ea4d4bad814c4edc654f5277f6197 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:52:30 -0700 Subject: [PATCH 32/93] add note about airplay (#6986) --- tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index f2334f1682d..2ccd44dd11d 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -246,6 +246,9 @@ By default, the server will listen on the following port mappings: | http | 5000 | | https | 5001 | +> [!WARNING] +> **MacOS** users should be aware that `airplay` by default utilizes port 5000. Ensure the port is free or use a non-default port as described below. + #### Environment Variable Set `ASPNETCORE_URLS` to define a custom port for either http or https (or both). Here are some examples: From 344a825e5667b7c22a54e6d12080d31ff38f6ec8 Mon Sep 17 00:00:00 2001 From: Albert Cheng <38804567+ckairen@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:02:52 -0700 Subject: [PATCH 33/93] [stress] test resource parallel deployment (#6695) * stress deploy pod waiting * testing * temp * revert temp changes * revert temp changes * revert temp changes * addons version bump * changelog update * index yaml for 0.2.2 --- .../stress-test-addons/CHANGELOG.md | 10 +++++++ .../kubernetes/stress-test-addons/Chart.yaml | 2 +- .../images/test-resource-deployer/Dockerfile | 4 +++ .../deploy-stress-test-resources.ps1 | 18 +++++++++++ .../kubernetes/stress-test-addons/index.yaml | 11 ++++++- .../templates/_init_deploy.tpl | 4 +++ .../templates/reader-role.yaml | 30 +++++++++++++++++++ 7 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/reader-role.yaml diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md index 573ac51d452..68d0adb99db 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md @@ -1,5 +1,15 @@ # Release History +## 0.2.2 (2023-09-21) + +### Features Added + +Added role binding for all pods. Allowing pods to have read access to information of other pods under the same namespace. + +### Bugs Fixed + +Fixed racing condition for ARM deployment when multiple pods run in parallel. + ## 0.2.1 (2023-09-14) ### Features Added diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml index d0292ce9a1c..6b69aeec5c5 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: stress-test-addons description: Baseline resources and templates for stress testing clusters -version: 0.2.1 +version: 0.2.2 appVersion: v0.1 diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile index a3f2bdd0a81..2f379a342f6 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile @@ -1,6 +1,10 @@ FROM mcr.microsoft.com/powershell RUN pwsh -c '$ErrorActionPreference = "Stop"; Install-Module Az -Force'; +RUN apt-get update && apt-get -y install curl +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +RUN install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +RUN kubectl version --client # For local testing, run prepare.ps1 before building the docker image COPY ./docker_build/common /common diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/deploy-stress-test-resources.ps1 b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/deploy-stress-test-resources.ps1 index 565857a2b45..16af8050dc9 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/deploy-stress-test-resources.ps1 +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/deploy-stress-test-resources.ps1 @@ -15,6 +15,24 @@ mkdir /azure Copy-Item "/scripts/stress-test/test-resources-post.ps1" -Destination "/azure/" Copy-Item "/mnt/testresources/*" -Destination "/azure/" +Write-Host "Job completion index $($env:JOB_COMPLETION_INDEX)" + +# Avoiding ARM deployment racing condition for multiple pods running in parallel +if ($env:JOB_COMPLETION_INDEX -and ($env:JOB_COMPLETION_INDEX -ne "0")) { + $cmd = "kubectl get pods -n $($env:NAMESPACE) -l job-name=$($env:JOB_NAME) -o jsonpath='{.items[?(@.metadata.annotations.batch\.kubernetes\.io/job-completion-index==`"0`")]..status.initContainerStatuses[?(@.name==`"init-azure-deployer`")].state.terminated.reason}'" + Write-Host $cmd + $result = "" + while ($result -ne "Completed") { + Write-Host "Waiting for pod index 0 deployment to complete." + Start-Sleep 10 + $result = Invoke-Expression $cmd + if ($LASTEXITCODE) { + Write-Host $result + throw "Failure getting pods" + } + } +} + # Capture output so we don't print environment variable secrets $env = & /common/TestResources/New-TestResources.ps1 ` -BaseName $env:BASE_NAME ` diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml index ea4f159ef06..4983be2ae3f 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml @@ -1,6 +1,15 @@ apiVersion: v1 entries: stress-test-addons: + - apiVersion: v2 + appVersion: v0.1 + created: "2023-09-21T14:55:43.2096162-07:00" + description: Baseline resources and templates for stress testing clusters + digest: 25da269ff8138e080a7703dcfb9833d6d772cb18470ef82262c4890c8b63eeda + name: stress-test-addons + urls: + - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.2.2.tgz + version: 0.2.2 - apiVersion: v2 appVersion: v0.1 created: "2023-09-14T17:52:24.713925164-04:00" @@ -181,4 +190,4 @@ entries: urls: - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.2.tgz version: 0.1.2 -generated: "2023-09-14T17:52:24.706686944-04:00" +generated: "2023-09-21T14:55:43.204981-07:00" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl index 4fc69e5d020..dd3bae9b6e5 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl @@ -18,6 +18,10 @@ value: {{ .Stress.ResourceGroupName }} - name: BASE_NAME value: {{ .Stress.BaseName }} + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: JOB_NAME + value: "{{ lower .Stress.Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}" volumeMounts: - name: "{{ .Release.Name }}-{{ .Release.Revision }}-test-resources" mountPath: /mnt/testresources diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/reader-role.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/reader-role.yaml new file mode 100644 index 00000000000..e8ae7cc5490 --- /dev/null +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/reader-role.yaml @@ -0,0 +1,30 @@ + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stress-k8s-reader-{{ .Release.Namespace }}-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - '*' + resources: + - 'pods' + - 'jobs' + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stress-k8s-reader-{{ .Release.Namespace }}-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +subjects: +- namespace: {{ .Release.Namespace }} + kind: ServiceAccount + name: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stress-k8s-reader-{{ .Release.Namespace }}-{{ .Release.Name }} From b9e245fe43e8ce6d4407fdf90116a315880c9bb3 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 22 Sep 2023 17:42:28 -0400 Subject: [PATCH 34/93] [stress] Add LockDeletionForDays parameter to set PodDisruptionBudget and cleanup job (#7002) * [stress] Add LockDeletionForDays parameter to set PodDisruptionBudget and cleanup job * [stress] Use matrix for parallel tests. PDB improvements+docs. * Fix kubectl namespace context preservation on login * Release addons --- .../stress-testing/deploy-stress-tests.ps1 | 5 +- .../stress-test-deployment-lib.ps1 | 37 ++++++--- tools/stress-cluster/chaos/README.md | 58 +++++++++----- .../network-stress-example/Chart.lock | 6 +- .../network-stress-example/Chart.yaml | 2 +- .../Chart.lock | 6 +- .../Chart.yaml | 2 +- .../examples/parallel-pod-example/Chart.lock | 6 +- .../examples/parallel-pod-example/Chart.yaml | 2 +- .../scenarios-matrix.yaml | 10 ++- .../templates/parallel-pod.yaml | 5 +- .../stress-debug-share-example/Chart.lock | 6 +- .../stress-debug-share-example/Chart.yaml | 2 +- .../stress-deployment-example/Chart.lock | 6 +- .../stress-deployment-example/Chart.yaml | 2 +- .../stress-test-addons/CHANGELOG.md | 12 +++ .../kubernetes/stress-test-addons/Chart.yaml | 2 +- .../kubernetes/stress-test-addons/index.yaml | 11 ++- .../templates/_pod_disruption_budget.tpl | 75 +++++++++++++++++++ .../templates/_stress_test.tpl | 51 ++++++------- 20 files changed, 222 insertions(+), 84 deletions(-) create mode 100644 tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_pod_disruption_budget.tpl diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 8abaa40d0cb..61d8f947d80 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -31,7 +31,10 @@ param( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + + # Prevent kubernetes from deleting nodes or rebalancing pods related to this test for N days + [Parameter(Mandatory=$False)][ValidateRange(1, 14)][int]$LockDeletionForDays ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index bdaec9e711a..e956508c439 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -59,7 +59,7 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushIm $kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json -AsHashtable $defaultNamespace = $null $targetContext = $kubeContext.contexts.Where({ $_.name -eq $clusterName }) | Select -First 1 - if ($targetContext -ne $null -and $targetContext.PSObject.Properties.Name -match "namespace") { + if ($targetContext -ne $null -and $targetContext.Contains('context') -and $targetContext.Contains('namespace')) { $defaultNamespace = $targetContext.context.namespace } @@ -107,7 +107,8 @@ function DeployStressTests( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + [Parameter(Mandatory=$False)][int]$LockDeletionForDays ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -168,7 +169,7 @@ function DeployStressTests( -subscription $subscription } - if ($FailedCommands.Count -lt $pkgs.Count) { + if ($FailedCommands.Count -lt $pkgs.Count -and !$Template) { Write-Host "Releases deployed by $deployer" Run helm list --all-namespaces -l deployId=$deployer } @@ -211,12 +212,14 @@ function DeployStressPackage( } $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} - Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" - $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} + if (!$Template) { + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" + $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + } $dockerBuildConfigs = @() @@ -317,8 +320,18 @@ function DeployStressPackage( $generatedConfigPath = Join-Path $pkg.Directory generatedValues.yaml $subCommand = $Template ? "template" : "upgrade" - $installFlag = $Template ? "" : "--install" - $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $installFlag, "--set", "stress-test-addons.env=$environment", "--values", $generatedConfigPath + $subCommandFlag = $Template ? "--debug" : "--install" + $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $subCommandFlag, "--values", $generatedConfigPath, "--set", "stress-test-addons.env=$environment" + + if ($LockDeletionForDays) { + $date = (Get-Date).AddDays($LockDeletionForDays).ToUniversalTime() + $isoDate = $date.ToString("o") + # Tell kubernetes job to run only on this specific future time. Technically it will run once per year. + $cron = "$($date.Minute) $($date.Hour) $($date.Day) $($date.Month) *" + + Write-Host "PodDisruptionBudget will be set to prevent deletion until $isoDate" + $helmCommandArg += "--set", "PodDisruptionBudgetExpiry=$($isoDate)", "--set", "PodDisruptionBudgetExpiryCron=$cron" + } $result = (Run @helmCommandArg) 2>&1 | Write-Host @@ -342,7 +355,7 @@ function DeployStressPackage( # Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around # specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources. # There is not currently support for setting these labels via the helm cli. - if(!$Template) { + if (!$Template) { $helmReleaseConfig = RunOrExitOnFailure kubectl get secrets ` -n $pkg.Namespace ` -l "status=deployed,name=$releaseName" ` diff --git a/tools/stress-cluster/chaos/README.md b/tools/stress-cluster/chaos/README.md index 3cec8266d72..2fb20f42d75 100644 --- a/tools/stress-cluster/chaos/README.md +++ b/tools/stress-cluster/chaos/README.md @@ -6,6 +6,7 @@ The chaos environment is an AKS cluster (Azure Kubernetes Service) with several * [Installation](#installation) * [Deploying a Stress Test](#deploying-a-stress-test) + * [Locking a test to run for a minimum number of days](#locking-a-test-to-run-for-a-minimum-number-of-days) * [Creating a Stress Test](#creating-a-stress-test) * [Layout](#layout) * [Stress Test Metadata](#stress-test-metadata) @@ -113,6 +114,27 @@ you can quick check the local logs: kubectl logs -n ``` +### Locking a test to run for a minimum number of days + +Occasionally the Kubernetes cluster can cause disruptions to long running tests. This will show up as a test pod +disappearing in the cluster (though all logs and other telemetry will still be available in app insights). This can +happen when nodes are auto-upgraded or scaled down to reduce resource usage. + +If a test must be run for a long time, it can be disruptive when a node reboot/shutdown happens. This can be prevented +by setting the `-LockDeletionForDays` parameter. When this parameter is set, the test pods will be deployed alongside a +[PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) that prevents nodes hosting the +pods from being removed. After the set number of days, this pod disruption budget will be deleted and the test will be +interruptable again. The test will not automatically shut down after this time, but it will no longer be locked. + +``` +/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -LockDeletionForDays 7 +``` + +To see when a pod's deletion lock will expire: + +``` +kubectl get pod -n -o jsonpath='{.metadata.annotations.deletionLockExpiry}' +``` ## Creating a Stress Test @@ -378,33 +400,31 @@ spec: #### Run multiple pods in parallel within a test job In some cases it may be necessary to run multiple instances of the same process/container in parallel as part of a test, -for example an eventhub test that needs to run 3 consumers, each in their own container. This can be achieved using -the `stress-test-addons.parallel-deploy-job-template.from-pod` template. The parallel feature leverages the +for example an eventhub test that needs to run 3 consumers, each in their own container. This can be achieved by adding +a `parallel` field in the matrix config. The parallel feature leverages the [job completion mode](https://kubernetes.io/docs/concepts/workloads/controllers/job/#completion-mode) feature. Test commands in the container can read the `JOB_COMPLETION_INDEX` environment variable to make decisions. For example, a messaging test that needs to run a single producer and multiple consumers can have logic that runs the producer when `JOB_COMPLETION_INDEX` is 0, and a consumer when it is not 0. -See the below example to enable parallel pods. Note the `(list . "stress.parallel-pod-example 3)` segment. The final argument (shown as `3` in the example) sets how many parallel pods should be run. - See a full working example of parallel pods [here](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/examples/parallel-pod-example). +See the below example to enable parallel pods via the matrix config (`scenarios-matrix.yaml`): + ``` -{{- include "stress-test-addons.parallel-deploy-job-template.from-pod" (list . "stress.parallel-pod-example" 3) -}} -{{- define "stress.parallel-pod-example" -}} -metadata: - labels: - testName: "parallel-pod-example" -spec: - containers: - - name: parallel-pod-example - image: busybox - command: ['bash', '-c'] - args: - - | - echo "Completed pod instance $JOB_COMPLETION_INDEX" - {{- include "stress-test-addons.container-env" . | nindent 6 }} -{{- end -}} +# scenarios-matrix.yaml +matrix: + scenarios: + parallel-example-a: + description: "Example for running multiple test containers in parallel" + # Adding this field into a matrix entry determines + # how many pods will run in parallel + parallel: 3 + parallel-example-b: + description: "Example for running multiple test containers in parallel" + parallel: 2 + non-parallel-example: + description: "This scenario is not run multiple pods in parallel" ``` NOTE: when multiple pods are run, each pod will invoke its own azure deployment init container. When many of these containers diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock index 28b9fa295cd..4699be0c47a 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.1 -digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 -generated: "2023-09-14T17:56:51.57311516-04:00" + version: 0.3.0 +digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e +generated: "2023-09-22T16:52:50.685996842-04:00" diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml index 05692aa8294..1ea43508b29 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock index 94a4c075909..c4c16285add 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.1 -digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 -generated: "2023-09-14T17:56:17.165505776-04:00" + version: 0.3.0 +digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e +generated: "2023-09-22T16:52:15.852268131-04:00" diff --git a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml index 7d6612c4965..22356e3c5da 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock index 127ef0fab41..5c055c98243 100644 --- a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.1 -digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 -generated: "2023-09-14T17:57:06.08946796-04:00" + version: 0.3.0 +digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e +generated: "2023-09-22T16:53:05.807496743-04:00" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml index d8813bfb611..9e0b08c613a 100644 --- a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml index 15eff91530f..f33715abda1 100644 --- a/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/scenarios-matrix.yaml @@ -1,4 +1,12 @@ matrix: scenarios: - parallel: + parallel-example-a: description: "Example for running multiple test containers in parallel" + # Adding this field into a matrix entry determines + # how many pods will run in parallel + parallel: 3 + parallel-example-b: + description: "Example for running multiple test containers in parallel" + parallel: 2 + non-parallel-example: + description: "This scenario is not run multiple pods in parallel" diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml index 8e51a8f36df..f522311a987 100644 --- a/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml +++ b/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml @@ -1,7 +1,4 @@ -{{- /* - The 3rd argument to this template (set as `3` below) is what determines the parallel pod count. -*/}} -{{- include "stress-test-addons.parallel-deploy-job-template.from-pod" (list . "stress.parallel-pod-example" 3) -}} +{{- include "stress-test-addons.deploy-job-template.from-pod" (list . "stress.parallel-pod-example") -}} {{- define "stress.parallel-pod-example" -}} metadata: labels: diff --git a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock index 62aa9eaba57..bf1b09e9471 100644 --- a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.1 -digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 -generated: "2023-09-14T17:56:41.783433241-04:00" + version: 0.3.0 +digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e +generated: "2023-09-22T16:52:39.169191153-04:00" diff --git a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml index 64df7b291b4..8891dbd3da2 100644 --- a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock index 0afd6de2436..e34172c455c 100644 --- a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock +++ b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.1 -digest: sha256:5128ca6ba7d0aedf802d7fa2a87d43d233741d00d673c971881c65cf0d4b6b78 -generated: "2023-09-14T17:56:01.788025676-04:00" + version: 0.3.0 +digest: sha256:3e21a7fdf5d6b37e871a6dd9f755888166fbb24802aa517f51d1d9223b47656e +generated: "2023-09-22T16:51:57.085186425-04:00" diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml index 3bbe19d54e8..59cb42f82e0 100644 --- a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml +++ b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.yaml @@ -10,5 +10,5 @@ annotations: dependencies: - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md index 68d0adb99db..b57b4e5fe49 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md @@ -1,5 +1,17 @@ # Release History +## 0.3.0 (2023-09-22) + +### Breaking Changes + +Move parallel job configuration into special matrix field `parallel` so that +parallelism can be set per scenario. Remove parallel-deploy-job-template way +of setting parallelism added in the 0.2.1 release. + +### Features Added + +Adds support for pod disruption budgets when helm values PodDisruptionBudgetExpiry and PodDisruptionBudgetExpiryCron are set. When the expiry is set, a pdb will be created matching all pods in a release, and a cron job will be created to clean up the pdb on a specified date. This allows users to mark a test as non-interruptable so that kubernetes will not shut down the node for upgrades, rebalancing, etc. + ## 0.2.2 (2023-09-21) ### Features Added diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml index 6b69aeec5c5..8f4499dc73c 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: stress-test-addons description: Baseline resources and templates for stress testing clusters -version: 0.2.2 +version: 0.3.0 appVersion: v0.1 diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml index 4983be2ae3f..8957d9e0c25 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml @@ -1,6 +1,15 @@ apiVersion: v1 entries: stress-test-addons: + - apiVersion: v2 + appVersion: v0.1 + created: "2023-09-22T16:48:47.835082288-04:00" + description: Baseline resources and templates for stress testing clusters + digest: 73d86e156b1f87d556ef3d51d048bb55f3f864867c9a422f0fd67bbc36a14c11 + name: stress-test-addons + urls: + - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.3.0.tgz + version: 0.3.0 - apiVersion: v2 appVersion: v0.1 created: "2023-09-21T14:55:43.2096162-07:00" @@ -190,4 +199,4 @@ entries: urls: - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.2.tgz version: 0.1.2 -generated: "2023-09-21T14:55:43.204981-07:00" +generated: "2023-09-22T16:48:47.827726695-04:00" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_pod_disruption_budget.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_pod_disruption_budget.tpl new file mode 100644 index 00000000000..570912373ae --- /dev/null +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_pod_disruption_budget.tpl @@ -0,0 +1,75 @@ +{{ define "stress-test-addons.pod-disruption-budget" }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + release: {{ .Release.Name }} +spec: + # Jobs do not implement `scale` otherwise we could set `minAvailable: 100%` or `maxUnavailable: 0` instead. + # Work around this by setting `minAvailable` to a number that will never be reached to simulate 100% + # so that the disruption budget will work in parallel pod scenarios (completionMode: indexed) + minAvailable: 10000 + selector: + matchLabels: + release: {{ .Release.Name }} +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: pdb-read-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pdb-read-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: ["*"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pdb-read-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pdb-read-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: pdb-read-{{ .Release.Name }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pdb-del-{{ substr 0 39 .Release.Name }}-{{ lower (randAlphaNum 3) }} + namespace: {{ .Release.Namespace }} +spec: + concurrencyPolicy: Forbid + schedule: "{{ .Values.PodDisruptionBudgetExpiryCron }}" + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + template: + spec: + serviceAccountName: pdb-read-{{ .Release.Name }} + restartPolicy: OnFailure + containers: + - name: kubectl + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ['bash', '-c'] + args: + - | + set -ex + curl -LOk "https://dl.k8s.io/release/$(curl -Lsk https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl version --client + kubectl delete poddisruptionbudgets -l release={{ .Release.Name }} +{{ end }} diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl index e058426b028..ab6178e0dc2 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_stress_test.tpl @@ -1,16 +1,9 @@ {{- define "stress-test-addons.job-wrapper.tpl" -}} +{{- $global := index . 0 -}} +{{- $definition := index . 1 -}} spec: template: - {{- include (index . 1) (index . 0) | nindent 4 -}} -{{- end -}} - -{{- define "stress-test-addons.parallel-job-wrapper.tpl" -}} -spec: - completions: {{ index . 2 }} - parallelism: {{ index . 2 }} - completionMode: Indexed - template: - {{- include (index . 1) (index . 0) | nindent 4 -}} + {{- include $definition $global | nindent 4 -}} {{- end -}} {{- define "stress-test-addons.deploy-job-template.tpl" -}} @@ -25,12 +18,21 @@ metadata: resourceGroupName: {{ .Stress.ResourceGroupName }} baseName: {{ .Stress.BaseName }} spec: + {{- if .Stress.parallel }} + completions: {{ .Stress.parallel }} + parallelism: {{ .Stress.parallel }} + completionMode: Indexed + {{- end }} backoffLimit: 0 template: metadata: labels: release: {{ .Release.Name }} scenario: {{ .Stress.Scenario }} + {{- if .Values.PodDisruptionBudgetExpiry }} + annotations: + deletionLockExpiry: {{ .Values.PodDisruptionBudgetExpiry }} + {{- end }} spec: # In cases where a stress test has higher resource requirements or needs a dedicated node, # a new nodepool can be provisioned and labeled to allow custom scheduling. @@ -65,22 +67,9 @@ spec: {{- toYaml (merge $jobOverride $tpl) -}} {{- end }} {{- include "stress-test-addons.static-secrets" $global }} -{{- end -}} - -{{- define "stress-test-addons.parallel-deploy-job-template.from-pod" -}} -{{- $global := index . 0 -}} -{{- $podDefinition := index . 1 -}} -{{- $parallel := index . 2 -}} -# Configmap template that adds the stress test ARM template for mounting -{{- include "stress-test-addons.deploy-configmap" $global }} -{{- range (default (list "stress") $global.Values.scenarios) }} ---- -{{ $jobCtx := fromYaml (include "stress-test-addons.util.mergeStressContext" (list $global . )) }} -{{- $jobOverride := fromYaml (include "stress-test-addons.parallel-job-wrapper.tpl" (list $jobCtx $podDefinition $parallel)) -}} -{{- $tpl := fromYaml (include "stress-test-addons.deploy-job-template.tpl" $jobCtx) -}} -{{- toYaml (merge $jobOverride $tpl) -}} +{{- if $global.Values.PodDisruptionBudgetExpiry }} +{{- include "stress-test-addons.pod-disruption-budget" $global }} {{- end }} -{{- include "stress-test-addons.static-secrets" $global }} {{- end -}} {{- define "stress-test-addons.env-job-template.tpl" -}} @@ -95,12 +84,21 @@ metadata: resourceGroupName: {{ .Stress.ResourceGroupName }} baseName: {{ .Stress.BaseName }} spec: + {{- if .Stress.parallel }} + completions: {{ .Stress.parallel }} + parallelism: {{ .Stress.parallel }} + completionMode: Indexed + {{- end }} backoffLimit: 0 template: metadata: labels: release: {{ .Release.Name }} scenario: {{ .Stress.Scenario }} + {{- if .Values.PodDisruptionBudgetExpiry }} + annotations: + deletionLockExpiry: {{ .Values.PodDisruptionBudgetExpiry }} + {{- end }} spec: nodeSelector: sku: 'default' @@ -128,4 +126,7 @@ spec: {{- toYaml (merge $jobOverride $tpl) -}} {{- end }} {{- include "stress-test-addons.static-secrets" $global }} +{{- if $global.Values.PodDisruptionBudgetExpiry }} +{{- include "stress-test-addons.pod-disruption-budget" $global }} +{{- end }} {{- end -}} From 5bc8c6517a1f0706e40b5e43fd89fe49eabe0366 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Fri, 22 Sep 2023 15:19:43 -0700 Subject: [PATCH 35/93] Update Error for Sanitizer Add (#7005) * create cherry-pickable commit * update tests to reflect new logged outputs --- .../test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs | 2 +- tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs index 843c590daef..44f2467c073 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs @@ -882,7 +882,7 @@ public async Task TestAddSanitizerThrowsOnMissingRequiredArgument() ); Assert.True(assertion.StatusCode.Equals(HttpStatusCode.BadRequest)); - Assert.Contains("Required parameter key System.String target was not found in the request body.", assertion.Message); + Assert.Contains("Required parameter key \"target\" was not found in the request body.", assertion.Message); } [Fact] diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs index f8d2f1ec7d3..f12f45d3c18 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs @@ -215,7 +215,7 @@ private object GenerateInstance(string typePrefix, string name, HashSet { if (!acceptableEmptyArgs.Contains(param.Name)) { - throw new HttpException(HttpStatusCode.BadRequest, $"Parameter {param.Name} was passed with no value. Please check the request body and try again."); + throw new HttpException(HttpStatusCode.BadRequest, $"Parameter \"{param.Name}\" was passed with no value. Please check the request body and try again."); } } @@ -229,7 +229,7 @@ private object GenerateInstance(string typePrefix, string name, HashSet } else { - throw new HttpException(HttpStatusCode.BadRequest, $"Required parameter key {param} was not found in the request body."); + throw new HttpException(HttpStatusCode.BadRequest, $"Required parameter key \"{param.Name}\" was not found in the request body."); } } } From 0adc8d379740c623d6985d8b97e330181a3a6b4d Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Mon, 25 Sep 2023 12:38:23 -0700 Subject: [PATCH 36/93] Create a pipeline for generic autorest verification (#6919) * Add autorest-preview pipeline --- .../templates/steps/sparse-checkout.yml | 11 +- .../scripts/Get-BuildSourceDescription.ps1 | 24 ++ .../Helpers/CommandInvocation-Helpers.ps1 | 42 ++ eng/common/scripts/New-RegenerateMatrix.ps1 | 102 +++++ .../scripts/TypeSpec-Project-Generate.ps1 | 23 +- eng/common/scripts/Update-GeneratedSdks.ps1 | 16 + eng/common/scripts/common.ps1 | 4 +- .../stages/archetype-autorest-preview.yml | 374 ++++++++++++++++++ .../steps/create-authenticated-npmrc.yml | 23 ++ .../steps/emit-pipeline-repositories.yml | 48 +++ .../autorest/New-EmitterPackageJson.ps1 | 50 +++ .../autorest/New-EmitterPackageLock.ps1 | 57 +++ 12 files changed, 753 insertions(+), 21 deletions(-) create mode 100644 eng/common/scripts/Get-BuildSourceDescription.ps1 create mode 100644 eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 create mode 100644 eng/common/scripts/New-RegenerateMatrix.ps1 create mode 100644 eng/common/scripts/Update-GeneratedSdks.ps1 create mode 100644 eng/pipelines/templates/stages/archetype-autorest-preview.yml create mode 100644 eng/pipelines/templates/steps/create-authenticated-npmrc.yml create mode 100644 eng/pipelines/templates/steps/emit-pipeline-repositories.yml create mode 100644 eng/scripts/autorest/New-EmitterPackageJson.ps1 create mode 100644 eng/scripts/autorest/New-EmitterPackageLock.ps1 diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index ab95453954c..448cb2c2e31 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -29,7 +29,7 @@ steps: if (!$dir) { $dir = "./$($repository.Name)" } - New-Item $dir -ItemType Directory -Force + New-Item $dir -ItemType Directory -Force | Out-Null Push-Location $dir if (Test-Path .git/info/sparse-checkout) { @@ -70,9 +70,14 @@ steps: # sparse-checkout commands after initial checkout will auto-checkout again if (!$hasInitialized) { - Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" + # Remove refs/heads/ prefix from branch names + $commitish = $repository.Commitish -replace '^refs/heads/', '' + + # use -- to prevent git from interpreting the commitish as a path + Write-Host "git -c advice.detachedHead=false checkout $commitish --" + # This will use the default branch if repo.Commitish is empty - git -c advice.detachedHead=false checkout $($repository.Commitish) + git -c advice.detachedHead=false checkout $commitish -- } else { Write-Host "Skipping checkout as repo has already been initialized" } diff --git a/eng/common/scripts/Get-BuildSourceDescription.ps1 b/eng/common/scripts/Get-BuildSourceDescription.ps1 new file mode 100644 index 00000000000..b0856101538 --- /dev/null +++ b/eng/common/scripts/Get-BuildSourceDescription.ps1 @@ -0,0 +1,24 @@ +param( + [string]$Variable, + [switch]$IsOutput +) + +$repoUrl = $env:BUILD_REPOSITORY_URI +$sourceBranch = $env:BUILD_SOURCEBRANCH + +$description = "[$sourceBranch]($repoUrl/tree/$sourceBranch)" +if ($sourceBranch -match "^refs/heads/(.+)$") { + $description = "Branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" +} elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $description = "Tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" +} elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $description = "Pull request: $repoUrl/pull/$($Matches[1])" +} + +if ($IsOutput) { + Write-Host "Setting output variable '$Variable' to '$description'" + Write-Host "##vso[task.setvariable variable=$Variable;isoutput=true]$description" +} else { + Write-Host "Setting variable '$Variable' to '$description'" + Write-Host "##vso[task.setvariable variable=$Variable]$description" +} diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 new file mode 100644 index 00000000000..5dc0c8c7da1 --- /dev/null +++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 @@ -0,0 +1,42 @@ +function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) +{ + $pipelineBuild = !!$env:TF_BUILD + $startTime = Get-Date + + if($pipelineBuild -and $GroupOutput) { + Write-Host "##[group]$Command" + } else { + Write-Host "> $Command" + } + + if($ExecutePath) { + Push-Location $ExecutePath + } + + try { + Invoke-Expression $Command + + $duration = (Get-Date) - $startTime + + if($pipelineBuild -and $GroupOutput) { + Write-Host "##[endgroup]" + } + + if($LastExitCode -ne 0) + { + if($pipelineBuild) { + Write-Error "##[error]Command failed to execute ($duration): $Command`n" + } else { + Write-Error "Command failed to execute ($duration): $Command`n" + } + } + else { + Write-Host "Command succeeded ($duration)`n" + } + } + finally { + if($ExecutePath) { + Pop-Location + } + } +} diff --git a/eng/common/scripts/New-RegenerateMatrix.ps1 b/eng/common/scripts/New-RegenerateMatrix.ps1 new file mode 100644 index 00000000000..1df97420c25 --- /dev/null +++ b/eng/common/scripts/New-RegenerateMatrix.ps1 @@ -0,0 +1,102 @@ +[CmdLetBinding()] +param ( + [Parameter()] + [string]$OutputDirectory, + + [Parameter()] + [string]$OutputVariableName, + + [Parameter()] + [int]$JobCount = 8, + + # The minimum number of items per job. If the number of items is less than this, then the number of jobs will be reduced. + [Parameter()] + [int]$MinimumPerJob = 10, + + [Parameter()] + [string]$OnlyTypespec +) + +. (Join-Path $PSScriptRoot common.ps1) + +[bool]$OnlyTypespec = $OnlyTypespec -in @("true", "t", "1", "yes", "y") + +# Divide the items into groups of approximately equal size. +function Split-Items([array]$Items) { + # given $Items.Length = 22 and $JobCount = 5 + # then $itemsPerGroup = 4 + # and $largeJobCount = 2 + # and $group.Length = 5, 5, 4, 4, 4 + $itemCount = $Items.Length + $jobsForMinimum = $itemCount -lt $MinimumPerJob ? 1 : [math]::Floor($itemCount / $MinimumPerJob) + + if ($JobCount -gt $jobsForMinimum) { + $JobCount = $jobsForMinimum + } + + $itemsPerGroup = [math]::Floor($itemCount / $JobCount) + $largeJobCount = $itemCount % $itemsPerGroup + $groups = [object[]]::new($JobCount) + + $i = 0 + for ($g = 0; $g -lt $JobCount; $g++) { + $groupLength = if ($g -lt $largeJobCount) { $itemsPerGroup + 1 } else { $itemsPerGroup } + $group = [object[]]::new($groupLength) + $groups[$g] = $group + for ($gi = 0; $gi -lt $groupLength; $gi++) { + $group[$gi] = $Items[$i++] + } + } + + Write-Host "$itemCount items split into $JobCount groups of approximately $itemsPerGroup items each." + + return , $groups +} + +# ensure the output directory exists +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + +if (Test-Path "Function:$GetDirectoriesForGenerationFn") { + $directoriesForGeneration = &$GetDirectoriesForGenerationFn +} +else { + $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory +} + +if ($OnlyTypespec) { + $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } +} + +[array]$packageDirectories = $directoriesForGeneration +| Sort-Object -Property FullName +| ForEach-Object { + [ordered]@{ + "PackageDirectory" = "$($_.Parent.Name)/$($_.Name)" + "ServiceArea" = $_.Parent.Name + } +} + +$batches = Split-Items -Items $packageDirectories + +$matrix = [ordered]@{} +for ($i = 0; $i -lt $batches.Length; $i++) { + $batch = $batches[$i] + $json = $batch.PackageDirectory | ConvertTo-Json -AsArray + + $firstPrefix = $batch[0].ServiceArea.Substring(0, 2) + $lastPrefix = $batch[-1].ServiceArea.Substring(0, 2) + + $key = "$firstPrefix`_$lastPrefix`_$i" + $fileName = "$key.json" + + Write-Host "`n`n==================================" + Write-Host $fileName + Write-Host "==================================" + $json | Out-Host + $json | Out-File "$OutputDirectory/$fileName" + + $matrix[$key] = [ordered]@{ "JobKey" = $key; "DirectoryList" = $fileName } +} + +$compressed = ConvertTo-Json $matrix -Depth 100 -Compress +Write-Output "##vso[task.setVariable variable=$OutputVariableName;isOutput=true]$compressed" diff --git a/eng/common/scripts/TypeSpec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 index e323f42aff6..05a0e0bdfd4 100644 --- a/eng/common/scripts/TypeSpec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -11,6 +11,7 @@ param ( $ErrorActionPreference = "Stop" . $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1 . $PSScriptRoot/common.ps1 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module @@ -21,38 +22,30 @@ function NpmInstallForProject([string]$workingDirectory) { Write-Host "Generating from $currentDur" if (Test-Path "package.json") { + Write-Host "Removing existing package.json" Remove-Item -Path "package.json" -Force } if (Test-Path ".npmrc") { + Write-Host "Removing existing .nprc" Remove-Item -Path ".npmrc" -Force } if (Test-Path "node_modules") { + Write-Host "Removing existing node_modules" Remove-Item -Path "node_modules" -Force -Recurse } if (Test-Path "package-lock.json") { + Write-Host "Removing existing package-lock.json" Remove-Item -Path "package-lock.json" -Force } - #default to root/eng/emitter-package.json but you can override by writing - #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1 $replacementPackageJson = Join-Path $PSScriptRoot "../../emitter-package.json" - if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { - $replacementPackageJson = &$GetEmitterPackageJsonPathFn - } Write-Host("Copying package.json from $replacementPackageJson") Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force - - #default to root/eng/emitter-package-lock.json but you can override by writing - #Get-${Language}-EmitterPackageLockPath in your Language-Settings.ps1 $emitterPackageLock = Join-Path $PSScriptRoot "../../emitter-package-lock.json" - if (Test-Path "Function:$GetEmitterPackageLockPathFn") { - $emitterPackageLock = &$GetEmitterPackageLockPathFn - } - $usingLockFile = Test-Path $emitterPackageLock if ($usingLockFile) { @@ -68,12 +61,10 @@ function NpmInstallForProject([string]$workingDirectory) { } if ($usingLockFile) { - Write-Host "> npm ci" - npm ci + Invoke-LoggedCommand "npm ci" } else { - Write-Host "> npm install" - npm install + Invoke-LoggedCommand "npm install" } if ($LASTEXITCODE) { exit $LASTEXITCODE } diff --git a/eng/common/scripts/Update-GeneratedSdks.ps1 b/eng/common/scripts/Update-GeneratedSdks.ps1 new file mode 100644 index 00000000000..dd671f6d8ad --- /dev/null +++ b/eng/common/scripts/Update-GeneratedSdks.ps1 @@ -0,0 +1,16 @@ +[CmdLetBinding()] +param( + [Parameter(Mandatory)] + [string]$PackageDirectoriesFile +) + +. $PSScriptRoot/common.ps1 +. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1 + +$ErrorActionPreference = 'Stop' + +if (Test-Path "Function:$UpdateGeneratedSdksFn") { + &$UpdateGeneratedSdksFn $PackageDirectoriesFile +} else { + Write-Error "Function $UpdateGeneratedSdksFn not implemented in Language-Settings.ps1" +} diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 39d65cdd681..cef0b23c562 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -60,8 +60,8 @@ $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme" $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions" $GetEmitterNameFn = "Get-${Language}-EmitterName" -$GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath" -$GetEmitterPackageLockPathFn = "Get-${Language}-EmitterPackageLockPath" +$GetDirectoriesForGenerationFn = "Get-${Language}-DirectoriesForGeneration" +$UpdateGeneratedSdksFn = "Update-${Language}-GeneratedSdks" # Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1 $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding" diff --git a/eng/pipelines/templates/stages/archetype-autorest-preview.yml b/eng/pipelines/templates/stages/archetype-autorest-preview.yml new file mode 100644 index 00000000000..862fca2b435 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-autorest-preview.yml @@ -0,0 +1,374 @@ +parameters: +# Whether to build alpha versions of the packages. This is passed as a flag to the build script. +- name: BuildAlphaVersion + type: boolean + +# Whether to use the `next` version of TypeSpec. This is passed as a flag to the init script. +- name: UseTypeSpecNext + type: boolean + +# The target to publish packages to. Currently supported values are 'internal' and 'public'. +- name: PublishTarget + type: string + +# Path to the emitter package's package.json file. If specified, it will be used to generate emitter-package.json in the artifact `build_artifacts`. +- name: EmitterPackageJsonPath + type: string + default: 'not-specified' + +# Custom steps to run after the autorest repository is cloned. If custom build steps are specified, the default init and build scripts will not be run. +# The build steps should produce the directory `/artifacts` with contents: +# packages/ +# autorest-csharp-2.0.0-alpha.4.tgz +# Microsoft.Azure.AutoRest.CSharp.2.0.0-alpha.4.nupkg +# typespec-csharp-1.2.3-alpha.4.tgz +- name: BuildSteps + type: stepList + default: [] + +# Custom steps to run after the sdk repository is cloned but before the generation matrix is created. +- name: SdkInitializationSteps + type: stepList + default: [] + +# List of packages to publish. Each package is an object with the following properties: +# name: The name of the package. This is used to determine the name of the file to publish. +# type: The type of package. Currently supported values are 'npm' and 'nuget'. +# file: The path to the file to publish. This is relative to the packages directory in the build artifacts directory. +- name: Packages + type: object + default: [] + +# Number of jobs to generate. This is the maximum number of jobs that will be generated. The actual number of jobs will be reduced if it would result in fewer than MinimumPerJob packages per job. +- name: RegenerationJobCount + type: number + default: 10 + +# Minimum number of packages to generate per job. +- name: MinimumPerJob + type: number + default: 10 + +stages: +- stage: Build + variables: + autorestRepositoryPath: $(Build.SourcesDirectory)/autorest + toolsRepositoryPath: $(Build.SourcesDirectory)/azure-sdk-tools + sdkRepositoryPath: $(Build.SourcesDirectory)/azure-sdk + jobs: + - job: Build + steps: + - template: ../steps/emit-pipeline-repositories.yml + parameters: + name: repositories + displayName: 'Get repository details' + + # Validate parameters and fail early if invalid + - ${{ if notIn(parameters.PublishTarget, 'internal', 'public') }}: + - script: | + echo "Publish target ${{ parameters.PublishTarget }} is not supported" + exit 1 + displayName: 'Unsupported PublishTarget' + condition: always() + + - ${{ each package in parameters.Packages }}: + - ${{ if notIn(package.type, 'npm', 'nuget') }}: + - script: | + echo "Package ${{ package.name }} has unsupported type: ${{ package.type }}" + exit 1 + displayName: 'Unsupported package type' + condition: always() + + - checkout: self + path: s/autorest + + - checkout: azure-sdk-tools + + - ${{ parameters.BuildSteps }} + + - ${{ if eq(length(parameters.BuildSteps), 0) }}: + - script: > + npm run ci-init -- --useTypeSpecNext ${{ parameters.UseTypeSpecNext }} + displayName: 'Run init script' + workingDirectory: $(autorestRepositoryPath) + + - script: > + npm run ci-build -- + --buildAlphaVersion ${{ parameters.BuildAlphaVersion }} + --buildNumber $(Build.BuildNumber) + --output $(Build.ArtifactStagingDirectory) + displayName: 'Run build script' + name: build_script + workingDirectory: $(autorestRepositoryPath) + + - ${{ if ne(parameters.EmitterPackageJsonPath, 'not-specified') }}: + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: $(toolsRepositoryPath)/eng/scripts/autorest/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '${{ parameters.EmitterPackageJsonPath }}' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(autorestRepositoryPath) + + - publish: $(Build.ArtifactStagingDirectory) + artifact: build_artifacts + displayName: Publish artifacts directory + + - pwsh: | + $branchName = 'autorest-failed-build-$(Build.BuildId)' + $repositoryName = '$(repositories.self.name)' + + . $(toolsRepositoryPath)/eng/common/scripts/git-branch-push.ps1 ` + -PRBranchName $branchName ` + -CommitMsg 'Update repo on failing build`n`nBuild url: $(System.CollectionUri)_build/results?buildId=$(Build.BuildId)' ` + -GitUrl "https://$(azuresdk-github-pat)@github.com/azure-sdk/$repositoryName.git" + + Write-Host "" + Write-Host @" + ##vso[task.logissue type=error]Created branch $branchName for build failure repro + + To clone the repo: + git clone https://github.com/azure-sdk/$repositoryName + + To add the remote to an existing clone: + git remote add azure-sdk https://github.com/azure-sdk/$repositoryName + git fetch azure-sdk + + To checkout the branch: + git checkout $branchName + "@ + + displayName: If failed, push changes + condition: failed() + workingDirectory: $(autorestRepositoryPath) + +- stage: Publish + dependsOn: Build + variables: + autorestRepositoryPath: $(Build.SourcesDirectory)/autorest + toolsRepositoryPath: $(Build.SourcesDirectory)/azure-sdk-tools + sdkRepositoryPath: $(Build.SourcesDirectory)/azure-sdk + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + jobs: + - job: Publish + steps: + - checkout: self + path: s/autorest + - checkout: azure-sdk-tools + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing + - ${{ if eq(parameters.PublishTarget, 'internal') }}: + - template: ../steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + - ${{ elseif eq(parameters.PublishTarget, 'public') }}: + - pwsh: | + "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' + displayName: Authenticate .npmrc for npmjs.org + workingDirectory: $(buildArtifactsPath)/packages + + # per package, publishing using appropriate tool + - ${{ each package in parameters.Packages }}: + - ${{ if eq(package.type, 'npm') }}: + - pwsh: | + $file = Resolve-Path "${{ package.file }}" + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + displayName: Publish ${{ package.name }} + workingDirectory: $(buildArtifactsPath)/packages + - ${{ elseif eq(package.type, 'nuget') }}: + - task: NuGetCommand@2 + displayName: Publish ${{ package.name }} + inputs: + command: 'push' + packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} + # Nuget packages are always published to the same internal feed. PublishTarget doesn't affect this. + nuGetFeedType: 'internal' + # Publish to https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net + publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + + - ${{ if ne(parameters.EmitterPackageJsonPath, 'not-specified') }}: + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: $(toolsRepositoryPath)/eng/scripts/autorest/New-EmitterPackageLock.ps1 + ${{ if eq(parameters.PublishTarget, 'internal') }}: + arguments: > + -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -NpmrcPath "$(buildArtifactsPath)/packages/.npmrc" + ${{ elseif eq(parameters.PublishTarget, 'public') }}: + arguments: > + -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - publish: $(Build.ArtifactStagingDirectory) + artifact: publish_artifacts + displayName: Publish artifacts directory + +- stage: Regenerate + dependsOn: + - Build + - Publish + variables: + autorestRepositoryPath: $(Build.SourcesDirectory)/autorest + toolsRepositoryPath: $(Build.SourcesDirectory)/azure-sdk-tools + sdkRepositoryPath: $(Build.SourcesDirectory)/azure-sdk + sdkRepositoryName: $[stageDependencies.Build.Build.outputs['repositories.sdk-repository.name']] + pullRequestTargetBranch: $[coalesce(stageDependencies.Build.Build.outputs['repositories.sdk-repository.branch'], 'main')] + sdkRepositoryCommitSha: $[stageDependencies.Build.Build.outputs['repositories.sdk-repository.version']] + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: auto-update-autorest-alpha-$(Build.BuildNumber) + jobs: + - job: Initialize + steps: + - template: ../../../common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + Repositories: + - Name: Azure/$(sdkRepositoryName) + WorkingDirectory: $(sdkRepositoryPath) + Commitish: $(sdkRepositoryCommitSha) + SkipCheckoutNone: true + - checkout: self + path: s/autorest + - checkout: azure-sdk-tools + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - download: current + artifact: publish_artifacts + displayName: Download pubish artifacts + + - ${{ if ne(parameters.EmitterPackageJsonPath, 'not-specified') }}: + - pwsh: | + Write-Host "Copying emitter-package.json to $(sdkRepositoryPath)/eng" + Copy-Item $(buildArtifactsPath)/emitter-package.json $(sdkRepositoryPath)/eng -Force + + Write-Host "Copying emitter-package-lock.json to $(sdkRepositoryPath)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(sdkRepositoryPath)/eng -Force + displayName: Copy emitter-package json files + + - ${{ parameters.SdkInitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(sdkRepositoryName) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(sdkRepositoryPath) + ScriptDirectory: $(toolsRepositoryPath)/eng/common/scripts + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + workingDirectory: $(sdkRepositoryPath) + filePath: $(sdkRepositoryPath)/eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.UseTypeSpecNext }} + + - publish: $(Build.ArtifactStagingDirectory) + artifact: matrix_artifacts + displayName: Publish matrix artifacts + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - checkout: self + - checkout: azure-sdk-tools + - template: ../../../common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + Repositories: + - Name: azure-sdk/$(sdkRepositoryName) + WorkingDirectory: $(sdkRepositoryPath) + Commitish: $(branchName) + SkipCheckoutNone: true + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + - download: current + artifact: publish_artifacts + displayName: Download pubish artifacts + - download: current + artifact: matrix_artifacts + displayName: Download matrix artifacts + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + workingDirectory: $(sdkRepositoryPath) + filePath: $(sdkRepositoryPath)/eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(sdkRepositoryName) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(sdkRepositoryPath) + ScriptDirectory: $(toolsRepositoryPath)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + steps: + - checkout: self + - checkout: azure-sdk-tools + + - task: PowerShell@2 + displayName: Get source description + inputs: + pwsh: true + filePath: $(toolsRepositoryPath)/eng/common/scripts/Get-BuildSourceDescription.ps1 + arguments: > + -Variable 'BuildSourceDescription' + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: $(toolsRepositoryPath)/eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner 'Azure' + -RepoName '$(sdkRepositoryName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle 'Autorest Regen Preview alpha-$(Build.BuildNumber) by $(Build.QueuedBy)' + -PRBody 'Triggered from $(BuildSourceDescription)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' diff --git a/eng/pipelines/templates/steps/create-authenticated-npmrc.yml b/eng/pipelines/templates/steps/create-authenticated-npmrc.yml new file mode 100644 index 00000000000..4b6f08359bd --- /dev/null +++ b/eng/pipelines/templates/steps/create-authenticated-npmrc.yml @@ -0,0 +1,23 @@ +parameters: + - name: npmrcPath + type: string + - name: registryUrl + type: string + +steps: +- pwsh: | + Write-Host "Creating .npmrc file ${{ parameters.npmrcPath }} for registry ${{ parameters.registryUrl }}" + $parentFolder = Split-Path -Path '${{ parameters.npmrcPath }}' -Parent + + if (!(Test-Path $parentFolder)) { + Write-Host "Creating folder $parentFolder" + New-Item -Path $parentFolder -ItemType Directory | Out-Null + } + + $content = "registry=${{ parameters.registryUrl }}`n`nalways-auth=true" + $content | Out-File '${{ parameters.npmrcPath }}' + displayName: 'Create .npmrc' +- task: npmAuthenticate@0 + displayName: Authenticate .npmrc + inputs: + workingFile: ${{ parameters.npmrcPath }} diff --git a/eng/pipelines/templates/steps/emit-pipeline-repositories.yml b/eng/pipelines/templates/steps/emit-pipeline-repositories.yml new file mode 100644 index 00000000000..645f8858426 --- /dev/null +++ b/eng/pipelines/templates/steps/emit-pipeline-repositories.yml @@ -0,0 +1,48 @@ +parameters: +- name: name + type: string +- name: displayName + type: string + +steps: +- pwsh: | + function Set-RepositoryVariable($key, $name, $value) { + Write-Host "Setting output variable ${{ parameters.name }}.$key.$name to $value" + Write-Host "##vso[task.setvariable variable=$key.$name;isOutput=true]$value" + } + + $accessToken = '$(System.AccessToken)' + $collectionUri = '$(System.CollectionUri)'.TrimEnd('/') + $project = '$(System.TeamProject)' + $buildDefinitionId = '$(System.DefinitionId)' + $buildId = '$(Build.BuildId)' + + $uri = "$collectionUri/$project/_apis/pipelines/$buildDefinitionId/runs/$buildId" + Write-Host "Getting pipeline run details from $uri" + + $pipeline = Invoke-RestMethod -Method GET $uri -Authentication Bearer -Token (ConvertTo-SecureString $accessToken -AsPlainText -Force) + + foreach ($repository in $pipeline.resources.repositories.psobject.properties) { + $version = $repository.Value.version + $refName = $repository.Value.refName + + if ($refName -match '^refs/heads/(.*)$') { + $branch = $Matches[1] + } + + if ($repository.Value.repository.fullName -match '^([^/]+)/([^/]+)$') { + $owner = $Matches[1] + $name = $Matches[2] + } + else { + $name = $repository.Value.repository.fullName + } + + Set-RepositoryVariable $repository.Name owner $owner + Set-RepositoryVariable $repository.Name name $name + Set-RepositoryVariable $repository.Name refName $refName + Set-RepositoryVariable $repository.Name branch $branch + Set-RepositoryVariable $repository.Name version $version + } + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} diff --git a/eng/scripts/autorest/New-EmitterPackageJson.ps1 b/eng/scripts/autorest/New-EmitterPackageJson.ps1 new file mode 100644 index 00000000000..a45206282b8 --- /dev/null +++ b/eng/scripts/autorest/New-EmitterPackageJson.ps1 @@ -0,0 +1,50 @@ +[CmdletBinding()] +param ( + [parameter(Mandatory = $true)] + [string]$PackageJsonPath, + + [parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [parameter(Mandatory = $false)] + [string]$PackageJsonFileName = "emitter-package.json" +) + +$knownPackages = @( + "@azure-tools/typespec-azure-core" + "@azure-tools/typespec-client-generator-core" + "@typespec/compiler" + "@typespec/eslint-config-typespec" + "@typespec/http" + "@typespec/rest" + "@typespec/versioning" +) + +$packageJson = Get-Content $PackageJsonPath | ConvertFrom-Json + +$devDependencies = @{} + +foreach ($package in $knownPackages) { + $pinnedVersion = $packageJson.devDependencies.$package + if ($pinnedVersion) { + $devDependencies[$package] = $pinnedVersion + } +} + +$emitterPackageJson = [ordered]@{ + "main" = "dist/src/index.js" + "dependencies" = @{ + $packageJson.name = $packageJson.version + } +} + +if($devDependencies.Keys.Count -gt 0) { + $emitterPackageJson["devDependencies"] = $devDependencies +} + +New-Item $OutputDirectory -ItemType Directory -ErrorAction SilentlyContinue | Out-Null +$OutputDirectory = Resolve-Path $OutputDirectory + +$dest = Join-Path $OutputDirectory $PackageJsonFileName +Write-Host "Generating $dest" +$emitterPackageJson | ConvertTo-Json -Depth 100 | Out-File $dest diff --git a/eng/scripts/autorest/New-EmitterPackageLock.ps1 b/eng/scripts/autorest/New-EmitterPackageLock.ps1 new file mode 100644 index 00000000000..e4ffb27edba --- /dev/null +++ b/eng/scripts/autorest/New-EmitterPackageLock.ps1 @@ -0,0 +1,57 @@ +[CmdletBinding()] +param ( + [parameter(Mandatory = $true)] + [string]$EmitterPackageJsonPath, + + [parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [parameter(Mandatory = $false)] + [string]$NpmrcPath, + + [parameter(Mandatory = $false)] + [string]$LockFileName = "emitter-package-lock.json" +) + +New-Item $OutputDirectory -ItemType Directory -ErrorAction SilentlyContinue | Out-Null +$OutputDirectory = Resolve-Path $OutputDirectory + +$tempFile = New-TemporaryFile +Remove-Item $tempFile + +# use a consistent folder name to avoid random package name in package-lock.json +Write-Host "Creating temporary folder $tempFile/emitter-consumer" +$tempFolder = New-Item "$tempFile/emitter-consumer" -ItemType Directory + +if ($NpmrcPath) { + Write-Host "Copy npmrc from $NpmrcPath to $tempFolder/.npmrc" + Copy-Item $NpmrcPath "$tempFolder/.npmrc" +} + +Push-Location $tempFolder + +try { + Write-Host "Copy $EmitterPackageJsonPath to $tempFolder/package.json" + Copy-Item $EmitterPackageJsonPath "$tempFolder/package.json" + + Write-Host 'npm install --legacy-peer-deps' + npm install --legacy-peer-deps + + if ($LASTEXITCODE) { + Write-Error "npm install failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } + + Write-Host '##[group]npm list --all --package-lock-only' + npm list --all --package-lock-only + Write-Host '##[endgroup]' + + $dest = Join-Path $OutputDirectory $LockFileName + Write-Host "Copy package-lock.json to $dest" + Copy-Item 'package-lock.json' $dest +} +finally { + Pop-Location +} + +Remove-Item $tempFolder -Recurse -Force From d91913d1db4b63f3c798e89e946d56896657b7b5 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 25 Sep 2023 17:18:27 -0700 Subject: [PATCH 37/93] Additional Documentation for `Matcher` and `Transform` concepts (#7009) * add a bunch of text around setting a matcher * spelling error correction --- .../Azure.Sdk.Tools.TestProxy/README.md | 78 +++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index 2ccd44dd11d..5ab1115386b 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -15,6 +15,8 @@ to documentation in your specific language repository in order to configure reco ## Table of contents - [Azure SDK Tools Test Proxy](#azure-sdk-tools-test-proxy) + - [Test documentation by language:](#test-documentation-by-language) + - [Table of contents](#table-of-contents) - [Installation](#installation) - [Via Local Compile or .NET](#via-local-compile-or-net) - [Via Docker Image](#via-docker-image) @@ -45,6 +47,10 @@ to documentation in your specific language repository in order to configure reco - [Session and Test Level Transforms, Sanitizers, and Matchers](#session-and-test-level-transforms-sanitizers-and-matchers) - [Add Sanitizer](#add-sanitizer) - [A note about where sanitizers apply](#a-note-about-where-sanitizers-apply) + - [Passing sanitizers in bulk](#passing-sanitizers-in-bulk) + - [Set a Matcher](#set-a-matcher) + - [The `Custom Default` Matcher](#the-custom-default-matcher) + - [Add a Transform](#add-a-transform) - [For Sanitizers, Matchers, or Transforms in general](#for-sanitizers-matchers-or-transforms-in-general) - [Viewing available/active Sanitizers, Matchers, and Transforms](#viewing-availableactive-sanitizers-matchers-and-transforms) - [To see customizations on a specific recording](#to-see-customizations-on-a-specific-recording) @@ -496,9 +502,13 @@ Of course, feel free to check any of the [examples](https://github.com/Azure/azu ## Session and Test Level Transforms, Sanitizers, and Matchers -A `sanitizer` is used to remove sensitive information prior to storage. When a request comes in during `playback` mode, the same set of `sanitizers` are applied prior to matching with the recordings. +The test-proxy is a record/playback solution. As a result, there a few concepts that devs will likely recognize from other record/playback solutions: -`Matchers` are used to retrieve a `RecordEntry` from a `RecordSession`. As of now, only a single matcher can be used when retrieving an entry during playback. +- A `Sanitizer` is used to remove sensitive information prior to storage. When a request comes in during `playback` mode, `sanitizers` are applied to the request prior to matching to a recording. +- `Matchers` are used to retrieve a request/response pair from a previous recording. By default, it functions by comparing `URI`, `Headers`, and `Body`. As of now, only a single matcher can be used when retrieving an entry during playback. +- A `Transform` is used when a user needs to "transform" a matched recording response with some value from the incoming request. This action is specific to `playback` mode. For instance, the test-proxy has two default `transforms`: + - `x-ms-client-id` is copied from request and applied to response prior to return. + - `x-ms-client-request-id` is copied from request and applied to response prior to return. Default sets of `matcher`, `transforms`, and `sanitizers` are applied during recording and playback. These default settings are all set at the `session` level. Customization is allowed for these default sets by accessing the `Admin` controller. @@ -582,12 +592,70 @@ In some cases, users need to register a lot (10+) of sanitizers. In this case, g ] ``` +### Set a Matcher + +Setting a matcher is just like adding a `sanitizer`. Set the `x-abstraction-identifier` value to the name of the matcher you want to instantiate, and provide the proper constructor arguments in the body! Check `Info/Available/` for available matchers. + +```jsonc +// POST to URI /Admin/SetMatcher +// headers +{ + "x-abstraction-identifier": "BodilessMatcher" +} +// request body is just empty JSON for a BodilessMatcher +{} +``` + +#### The `Custom Default` Matcher + +The default `matcher` for test-proxy is usable in _most_ situations. `Sanitizers` can be used to clear non-available information prior to saving to disk, and playback tests should ensure that they match the expected values in the recordings. Sanization can usually eliminate the need for custom matching, but in some circumstances this is just not feasible. + +If a dev must only change a _couple_ elements of the default matcher, the `CustomDefault` matcher is the way to go! + +```jsonc +// POST to URI /Admin/SetMatcher +// headers +{ + "x-abstraction-identifier": "CustomDefaultMatcher" +} +// request body +{ + // Should the body value be compared during lookup operations? + "compareBodies": true, + // A comma separated list of additional headers that should be excluded during matching. "Excluded" headers are entirely ignored. + "excludedHeaders": "traceparent", + // A comma separated list of additional headers that should be ignored during matching. Any headers that are "ignored" will not do value comparison when matching. + // The "presence" of the headers will still be checked however. + "ignoredHeaders": "User-Agent, Origin, Content-Length", + // By default, the test-proxy does not sort query params before matching. Setting true will sort query params alphabetically before comparing URI. + "ignoredQueryOrdering": false, + // A comma separated list of query parameterse that should be ignored during matching. + "ignoredQueryParameters" "token" +} +``` + +### Add a Transform + +Just like `sanitizers` and `matchers`, select an abstraction-id in header, provide a json body with any arguments/details. + +```jsonc +// POST to URI /Admin/AddTransform +// headers +{ + "x-abstraction-identifier": "ApiVersionTransform" +} +// empty request body for ApiVersionTransform +{} +``` + +This will add a transform that copies `api-version` header from request onto the matched response during `playback`. + ### For Sanitizers, Matchers, or Transforms in general When invoked as basic requests to the `Admin` controller, these settings will be applied to **all** further requests and responses. Both `Playback` and `Recording`. Where applicable. -- `sanitizers` are applied before the recording is saved to disk AND when an incoming request is matched. -- A custom `matcher` can be set for a session or individual recording and is applied when retrieving an entry from a loaded recording. +- `sanitizers` are applied before the recording is saved to disk AND on an incoming request prior to the `match` operation. +- A custom `matcher` can be set for a session or individual recording and is applied when retrieving an entry from a loaded recording. Only a single matcher can be honored when matching requests to recorded entries. Registering two matchers one after another will simply result in matching using the last matcher provided. - `transforms` are applied when returning a request during playback. Currently, the configured set of transforms/playback/sanitizers are NOT propogated onto disk alongside the recording. @@ -596,7 +664,7 @@ Currently, the configured set of transforms/playback/sanitizers are NOT propogat Launch the test-proxy through your chosen method, then visit: -- `/Info/Available` to see all available +- `/Info/Available` to see all available. - `/Info/Active` to see all currently active for all sessions. Note that the `constructor arguments` that are documented must be present (where documented as such) in the body of the POST sent to the Admin Interface. From d2b0f9838e57867e81a0aec8685ac658703cd30b Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 26 Sep 2023 21:29:16 -0400 Subject: [PATCH 38/93] [stress] Change prod maintenance day. Fix stress watcher deploy (#7013) --- .../scripts/stress-testing/stress-test-deployment-lib.ps1 | 2 +- tools/stress-cluster/cluster/azure/cluster/cluster.bicep | 3 ++- tools/stress-cluster/cluster/azure/main.bicep | 2 ++ tools/stress-cluster/cluster/azure/parameters/prod.json | 3 +++ .../cluster/kubernetes/stress-infrastructure/Chart.lock | 6 +++--- .../cluster/kubernetes/stress-infrastructure/Chart.yaml | 2 +- .../stress-infrastructure/templates/stresswatcher.yaml | 2 ++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index e956508c439..dde43649a39 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -59,7 +59,7 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushIm $kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json -AsHashtable $defaultNamespace = $null $targetContext = $kubeContext.contexts.Where({ $_.name -eq $clusterName }) | Select -First 1 - if ($targetContext -ne $null -and $targetContext.Contains('context') -and $targetContext.Contains('namespace')) { + if ($targetContext -ne $null -and $targetContext.Contains('context') -and $targetContext.context.Contains('namespace')) { $defaultNamespace = $targetContext.context.namespace } diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep index 84ec0aafecc..8106327adc5 100644 --- a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep +++ b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep @@ -6,6 +6,7 @@ param clusterName string param location string = resourceGroup().location param defaultAgentPoolMinNodes int = 6 param defaultAgentPoolMaxNodes int = 20 +param maintenanceWindowDay string = 'Monday' // AKS does not allow agentPool updates via existing managed cluster resources param updateNodes bool = false @@ -100,7 +101,7 @@ resource maintenanceConfig 'Microsoft.ContainerService/managedClusters/maintenan startTime: '02:00' schedule: { weekly: { - dayOfWeek: 'Monday' + dayOfWeek: maintenanceWindowDay intervalWeeks: 1 } } diff --git a/tools/stress-cluster/cluster/azure/main.bicep b/tools/stress-cluster/cluster/azure/main.bicep index a35840a96cd..1abe4033728 100644 --- a/tools/stress-cluster/cluster/azure/main.bicep +++ b/tools/stress-cluster/cluster/azure/main.bicep @@ -9,6 +9,7 @@ param staticTestKeyvaultGroup string param monitoringLocation string = 'centralus' param defaultAgentPoolMinNodes int = 6 param defaultAgentPoolMaxNodes int = 20 +param maintenanceWindowDay string = 'Monday' param tags object // AKS does not allow agentPool updates via existing managed cluster resources param updateNodes bool = false @@ -79,6 +80,7 @@ module cluster 'cluster/cluster.bicep' = { clusterName: clusterName defaultAgentPoolMinNodes: defaultAgentPoolMinNodes defaultAgentPoolMaxNodes: defaultAgentPoolMaxNodes + maintenanceWindowDay: maintenanceWindowDay tags: tags groupSuffix: groupSuffix workspaceId: logWorkspace.outputs.id diff --git a/tools/stress-cluster/cluster/azure/parameters/prod.json b/tools/stress-cluster/cluster/azure/parameters/prod.json index 1b89cdd2ad7..b9565c38566 100644 --- a/tools/stress-cluster/cluster/azure/parameters/prod.json +++ b/tools/stress-cluster/cluster/azure/parameters/prod.json @@ -29,6 +29,9 @@ "defaultAgentPoolMaxNodes": { "value": 10 }, + "maintenanceWindowDay": { + "value": "Friday" + }, "tags": { "value": { "environment": "Prod", diff --git a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock index 4b7663d20f1..f2c2231f843 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock +++ b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock @@ -4,6 +4,6 @@ dependencies: version: 2.1.4 - name: stress-test-addons repository: https://stresstestcharts.blob.core.windows.net/helm/ - version: 0.2.0 -digest: sha256:f9f4b6fa01e634fe3842ed1d257a2618483b1f8972ed85b97fb9abf641c8084a -generated: "2022-11-15T20:35:48.4869487-05:00" + version: 0.3.0 +digest: sha256:dc42ee5bb2c0b0427c47ae898ddab359e05a3e224ea22002175f2d74d95a469b +generated: "2023-09-25T21:27:50.950627239-04:00" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.yaml b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.yaml index 3dbe11a6dca..9054573f7e6 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.yaml @@ -29,5 +29,5 @@ dependencies: repository: https://charts.chaos-mesh.org condition: deploy.chaosmesh - name: stress-test-addons - version: ~0.2.0 + version: ~0.3.0 repository: "@stress-test-charts" diff --git a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher.yaml b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher.yaml index dae909fde90..53713d26fa2 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher.yaml +++ b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher.yaml @@ -34,3 +34,5 @@ spec: {{- include "stress-test-addons.env-volumes" $ctx | nindent 8 }} # Volume template for mounting azure file share for debugging {{- include "stress-test-addons.debug-file-volumes" $ctx | nindent 8 }} + +{{- include "stress-test-addons.static-secrets" . }} From ddc4e1090d39470d7100ccef68d1992f57f5f939 Mon Sep 17 00:00:00 2001 From: Rodge Fu Date: Wed, 27 Sep 2023 17:49:03 +0800 Subject: [PATCH 39/93] some improvements per feedback (#7018) * some improvements 1. make CHANGELOG.MD to CHANGELOG.md for case-sensitive environment 2. move spec version bump from Other Changes to Features Added group * small improve --- .../mergedChangelog1.md | 5 ++++- .../ChangeLogResult.cs | 8 ++++++-- .../Context.cs | 2 +- .../Report/Release.cs | 17 ++++++++++------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md index c4204f3f239..80398308fb4 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md @@ -18,9 +18,12 @@ - Removed property method 'Set' for 'String PropertyToChangeToGet' in type Azure.ResourceManager.AppService.TestProperty - Removed type 'Azure.ResourceManager.AppService.TypeToBeDeleted' -### Other Changes +### Features Added - spec upgraded + +### Other Changes + - Azure Core upgraded - Azure RM upgraded - Obsoleted method 'Void StaticMethodToBeObsoleted()' in type Azure.ResourceManager.AppService.StaticTypeToBeObsoleted diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/ChangeLogResult.cs b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/ChangeLogResult.cs index 87ab76ab63d..08c91862519 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/ChangeLogResult.cs +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/ChangeLogResult.cs @@ -31,9 +31,13 @@ public Release GenerateReleaseNote(string version, string date, List 0) + report.Groups.Add(featureAddedGroup); + + ReleaseNoteGroup othersGroup = new ReleaseNoteGroup("Other Changes"); if (AzureCoreVersionChange != null) othersGroup.Notes.Add(new ReleaseNote(AzureCoreVersionChange.Description, PREFIX)); if (AzureResourceManagerVersionChange != null) diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Context.cs b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Context.cs index d2aeee90ba8..a73395ee18d 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Context.cs +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Context.cs @@ -49,7 +49,7 @@ public class Context public string AzureCoreChangeLogGithubKey => "sdk/core/Azure.Core/CHANGELOG.md"; public string AzureCoreChangeLogMdFile => Path.Combine(RepoRoot, AzureCoreChangeLogGithubKey); - public string AzureResourceManagerChangeLogGithubKey => "sdk/resourcemanager/Azure.ResourceManager/CHANGELOG.MD"; + public string AzureResourceManagerChangeLogGithubKey => "sdk/resourcemanager/Azure.ResourceManager/CHANGELOG.md"; public string AzureResourceManagerChangeLogMdFile => Path.Combine(RepoRoot, AzureResourceManagerChangeLogGithubKey); public bool IsPreview => Helper.IsPreviewRelease(this.ReleaseVersion); diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Report/Release.cs b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Report/Release.cs index db875237a8d..07a7ae6d274 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Report/Release.cs +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen/Report/Release.cs @@ -70,7 +70,7 @@ private void MergeByOverwrite(Release to) private void MergeByGrup(Release to) { - foreach(var fromGroup in this.Groups) + foreach (var fromGroup in this.Groups) { var found = to.Groups.FirstOrDefault(g => g.Name == fromGroup.Name); if (found != null) @@ -81,21 +81,24 @@ private void MergeByGrup(Release to) private void MergeByLine(Release to) { - foreach(var fromGroup in this.Groups) + foreach (var fromGroup in this.Groups) { var toGroup = to.Groups.FirstOrDefault(g => string.Equals(g.Name, fromGroup.Name, StringComparison.OrdinalIgnoreCase)); - if(toGroup == null) + if (toGroup == null) { to.Groups.Add(fromGroup); } else { - foreach(var fromItem in fromGroup.Notes) + int indexToInsert = toGroup.Notes.FindLastIndex(n => !string.IsNullOrEmpty(n.Note)) + 1; + int lastNonEmptyIndex = fromGroup.Notes.FindLastIndex(n => !string.IsNullOrEmpty(n.Note)); + for (int i = lastNonEmptyIndex; i >= 0; i--) { + var fromItem = fromGroup.Notes[i]; var toItem = toGroup.Notes.FirstOrDefault(t => string.Equals(fromItem.ToString(), t.ToString(), StringComparison.OrdinalIgnoreCase)); - if(toItem == null) + if (toItem == null) { - toGroup.Notes.Add(fromItem); + toGroup.Notes.Insert(indexToInsert, fromItem); } else { @@ -133,7 +136,7 @@ public static List FromChangelog(string changelog) Release curRelease = firstRelease!; ReleaseNoteGroup curGroup = new ReleaseNoteGroup(""); - for(i = i+1; i < lines.Length; i++) + for (i = i + 1; i < lines.Length; i++) { if (ReleaseNoteGroup.TryParseGroupTitle(lines[i], out ReleaseNoteGroup? newGroup)) { From 02476be3e84d7b8d925bd403d4b76715dee55bf2 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Wed, 27 Sep 2023 10:15:24 -0700 Subject: [PATCH 40/93] Fix .NET branding in public docs (#6925) * Fix .NET branding in public docs Was an attempt to try fixing issue #6924 but it seems that code may live elsewhere. Figured I might as well update public-facing first-party branding in this repo while I was in here. * Update src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md Co-authored-by: Mariana Rios Flores * Resolve PR feedback --------- Co-authored-by: Mariana Rios Flores --- packages/python-packages/doc-warden/README.md | 2 +- src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md | 4 ++-- src/dotnet/APIView/apiview.yml | 2 +- tools/assets-automation/asset-sync/README.md | 4 ++-- tools/http-fault-injector/README.md | 2 +- .../Azure.SDK.Management.ChangelogGen.Tests/changelog1.md | 2 +- .../mergedChangelog1.md | 2 +- tools/perf-automation/README.md | 2 +- tools/sdk-generation-pipeline/documents/docker/README.md | 2 +- .../documents/docker/vscode-connect-docker-container.md | 2 +- tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md | 2 +- tools/test-proxy/CONTRIBUTING.md | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/python-packages/doc-warden/README.md b/packages/python-packages/doc-warden/README.md index ba08a87e84c..78906e9f937 100644 --- a/packages/python-packages/doc-warden/README.md +++ b/packages/python-packages/doc-warden/README.md @@ -224,7 +224,7 @@ the `package_indexing_exclusion_list` array members to enable just this sort of `package_indexing_traversal_stops` is used during parse of .NET language repos _only_. This is due to how the discovery logic for readme and changelog is implemented for .NET projects. Specifically, readmes for a .csproj are often a couple directories up from their parent .csproj location! -For .net, `warden` will traverse **up** one directory at a time, looking for the readme and changelog files in each traversed directory. `warden` will continue to traverse until... +For .NET, `warden` will traverse **up** one directory at a time, looking for the readme and changelog files in each traversed directory. `warden` will continue to traverse until... 1. It discovers a folder with a `.sln` within it 2. It encounters a folder that exactly matches one present in `package_indexing_traversal_stops` diff --git a/src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md b/src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md index fe5951aeda2..56c804ccbde 100644 --- a/src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md +++ b/src/dotnet/APIView/APIViewWeb/CONTRIBUTING.md @@ -14,7 +14,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio ## Where to begin -Core of the APIView tool is the web app developed using ASP.Net and TypeScript. This core module takes care of presenting reviews to users, storing review files and metadata in Azure Storage Account and Cosmos database and process requests from Azure Devops pipelines and respond. We also have language level parsers that converts each language specific artifact into a common json stub file that's known to core APIView web app. So, first step as a contributor is to understand the feature or bug fix you would like to submit and identify the area you would like to contribute to. Language parsers are either added as plugin modules developed in .Net or developed using corresponding language as command line tool to extract and generate stub file. If change is specific to a language in how langauge specific API details are extracted to stub tokens then change will be at parser level for that language. If change is applicable for all languages then change will most likely be in core APIView web app. +Core of the APIView tool is the web app developed using ASP.NET and TypeScript. This core module takes care of presenting reviews to users, storing review files and metadata in Azure Storage Account and Cosmos database and process requests from Azure Devops pipelines and respond. We also have language level parsers that converts each language specific artifact into a common json stub file that's known to core APIView web app. So, first step as a contributor is to understand the feature or bug fix you would like to submit and identify the area you would like to contribute to. Language parsers are either added as plugin modules developed in .NET or developed using corresponding language as command line tool to extract and generate stub file. If change is specific to a language in how langauge specific API details are extracted to stub tokens then change will be at parser level for that language. If change is applicable for all languages then change will most likely be in core APIView web app. | Module | Source Path | @@ -39,7 +39,7 @@ Following are tools required to develop and run test instance of APIView to veri - Git - Visual Studio -- .Net +- .NET - Any LTS version of Node.js [download](https://nodejs.org/en/download/) - Java (Optional: Only required if you want to generate Java review locally) - Python 3.9+ (Optional: Only required if you want to generate Python review locally) diff --git a/src/dotnet/APIView/apiview.yml b/src/dotnet/APIView/apiview.yml index 6b7f91ab963..d94f67c5112 100644 --- a/src/dotnet/APIView/apiview.yml +++ b/src/dotnet/APIView/apiview.yml @@ -172,7 +172,7 @@ stages: - pwsh: | dotnet --list-runtimes dotnet --version - displayName: 'List .Net run times' + displayName: 'List .NET run times' - task: GoTool@0 inputs: diff --git a/tools/assets-automation/asset-sync/README.md b/tools/assets-automation/asset-sync/README.md index b849a80b903..083a77cfe30 100644 --- a/tools/assets-automation/asset-sync/README.md +++ b/tools/assets-automation/asset-sync/README.md @@ -151,8 +151,8 @@ TODO: Read about this option. - The physical drive the enlistment is on seems to matter more when using LFS - For example: two machines, both physically in Redmond and on corpnet, one was a single spinning drive and the other was an array, the clone time difference was negligible (30 vs 22 seconds) but the checkout times were wildly variant (18 seconds vs 2 minutes and 58 seconds) - Distance matters – Fetching from Redmond produced very different numbers than fetching from Australia. On average clone times were double of what we’d see on corpnet and still around a minute, but the checkout times were horrendous. Non-LFS checkout had an average of 32 seconds where the LFS checkouts were taking an average of 8 minutes. Note: This was on an SSD, not a spinning disk. - Git LFS files are pulled individually - There’s no way to bulk pull everything. The files are pulled over https with a default concurrency of 3 and while this is something that could be tweaked in the lfs config it would potentially help checkout times on an SSD but make the times on a spinning disk worse. -- The size on disk size, for us, would initially get worse and then eventually taper off – Because we wouldn’t be rewriting history we’re still going to have space in .git/objects from the previous versions of these files but with the way LFS works, you’re also going to have .git/lfs/objects. Unlike .git/objects, the .git/lfs/objects will only contain the version that’s been pulled local. If you checkout a version you don’t have, it’ll update the .git/lfs/object with that and the other version will simply be a file with pointer. In the case of the .net repo, which has 6029 recording json files taking up about 671MB, this means that the size on disk would grow by that amount. The reason for this bloat is that there’s no compression on the LFS folder. - +- The size on disk size, for us, would initially get worse and then eventually taper off – Because we wouldn’t be rewriting history we’re still going to have space in .git/objects from the previous versions of these files but with the way LFS works, you’re also going to have .git/lfs/objects. Unlike .git/objects, the .git/lfs/objects will only contain the version that’s been pulled local. If you checkout a version you don’t have, it’ll update the .git/lfs/object with that and the other version will simply be a file with pointer. In the case of the .NET repo, which has 6029 recording json files taking up about 671MB, this means that the size on disk would grow by that amount. The reason for this bloat is that there’s no compression on the LFS folder. + ### `External Git Repo` Current prototype visible [here.](./assets.ps1). diff --git a/tools/http-fault-injector/README.md b/tools/http-fault-injector/README.md index 8caba990dfc..d391a9c3819 100644 --- a/tools/http-fault-injector/README.md +++ b/tools/http-fault-injector/README.md @@ -7,7 +7,7 @@ * No response. Then either wait indefinitely, close (TCP FIN), or abort (TCP RST). ## Installation -1. [Install .Net](https://dotnet.microsoft.com/download) +1. [Install .NET](https://dotnet.microsoft.com/download) 2. Install http-fault-injector ``` diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/changelog1.md b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/changelog1.md index 77c6f4bbbb5..a7880fcd455 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/changelog1.md +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/changelog1.md @@ -42,7 +42,7 @@ Polishing since last public beta release: - Prepended `AppService` prefix to all single / simple model names. - Corrected the format of all `ResourceIdentifier` type properties / parameters. - Corrected the format of all `AzureLocation` type properties / parameters. -- Corrected all acronyms that not follow [.Net Naming Guidelines](https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-guidelines). +- Corrected all acronyms that not follow [.NET Naming Guidelines](https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-guidelines). - Corrected enumeration name by following [Naming Enumerations Rule](https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces#naming-enumerations). - Corrected the suffix of `DateTimeOffset` properties / parameters. - Corrected the name of interval / duration properties / parameters that end with units. diff --git a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md index 80398308fb4..307502a436e 100644 --- a/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md +++ b/tools/net-changelog-gen-mgmt/Azure.SDK.Management.ChangelogGen.Tests/mergedChangelog1.md @@ -68,7 +68,7 @@ Polishing since last public beta release: - Prepended `AppService` prefix to all single / simple model names. - Corrected the format of all `ResourceIdentifier` type properties / parameters. - Corrected the format of all `AzureLocation` type properties / parameters. -- Corrected all acronyms that not follow [.Net Naming Guidelines](https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-guidelines). +- Corrected all acronyms that not follow [.NET Naming Guidelines](https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-guidelines). - Corrected enumeration name by following [Naming Enumerations Rule](https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces#naming-enumerations). - Corrected the suffix of `DateTimeOffset` properties / parameters. - Corrected the name of interval / duration properties / parameters that end with units. diff --git a/tools/perf-automation/README.md b/tools/perf-automation/README.md index bbc49b3236f..399223106ea 100644 --- a/tools/perf-automation/README.md +++ b/tools/perf-automation/README.md @@ -4,7 +4,7 @@ `perf-automation` is a tool to run multiple perf tests in any languages and collect the results in a single JSON file. ## Walkthrough -1. [Install .Net](https://dotnet.microsoft.com/download) +1. [Install .NET](https://dotnet.microsoft.com/download) 2. `git clone https://github.com/Azure/azure-sdk-tools` diff --git a/tools/sdk-generation-pipeline/documents/docker/README.md b/tools/sdk-generation-pipeline/documents/docker/README.md index 5eae05240e5..0f22e3916c1 100644 --- a/tools/sdk-generation-pipeline/documents/docker/README.md +++ b/tools/sdk-generation-pipeline/documents/docker/README.md @@ -36,7 +36,7 @@ Parameter description: | { local_autorest_config } | Optional. When you generate data-plane sdk, and there is no autorest configuration in sdk repository or you want to change the autorest configuration, you can set new autorest config in a file and mount it to the docker container. About the content of file, please refer to [document](https://github.com/Azure/azure-rest-api-specs/blob/dpg-doc/documentation/onboard-dpg-in-sdkautomation/add-autorest-configuration-in-spec-comment.md) | /home/test/autorest.md ([Example file](./autorest-config-file-sample.md)) | | { relative_readme } | Required. It's used to specify the readme.md file and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md | | { relative_typespec_project } | Required. It's used to specify the typespec project folder and docker image uses it to generate SDKs. it's the relative path from {path_to_local_spec_repo} | specification/agrifood/resource-manager/readme.md | -| { sdk_to_generate } | Required. It's used to specify which language of sdk you want to generate. Supported value for management sdk: js, java, python, .net, and go. Supported value for dataplane sdk: js, java, python, and .net. If you want to generate multi-packages, use comma to separate them. (__Not recommend to generate multi packages in one docker container because the docker will failed when encoutering error in generating one sdk, and the remaining sdk will be generated__) | js,java | +| { sdk_to_generate } | Required. It's used to specify which language of sdk you want to generate. Supported value for management sdk: js, java, python, .NET, and go. Supported value for dataplane sdk: js, java, python, and .NET. If you want to generate multi-packages, use comma to separate them. (__Not recommend to generate multi packages in one docker container because the docker will failed when encoutering error in generating one sdk, and the remaining sdk will be generated__) | js,java | Example Command: 1. Generate codes with Autorest: diff --git a/tools/sdk-generation-pipeline/documents/docker/vscode-connect-docker-container.md b/tools/sdk-generation-pipeline/documents/docker/vscode-connect-docker-container.md index 9b09fe11926..8261f0c0d3b 100644 --- a/tools/sdk-generation-pipeline/documents/docker/vscode-connect-docker-container.md +++ b/tools/sdk-generation-pipeline/documents/docker/vscode-connect-docker-container.md @@ -11,7 +11,7 @@ Please follow the following steps to connect your vscode to docker container. 1. Press `F1` and select `Remote-Containers: Attach to Running Container`. 2. Select your running docker image, and attach to it. 3. After vscode connects to docker container, open folder `/work-dir/{sdk-repository}`. - 1. For .Net, you can only open the generated SDK namespace folder, such as `Azure.Verticals.AgriFood.Farming`. + 1. For .NET, you can only open the generated SDK namespace folder, such as `Azure.Verticals.AgriFood.Farming`. 2. For Java, you can only open the generated package, such as `azure-resourcemanager-agrifood`. Then you can write your codes in vscode. diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index 5ab1115386b..0e401a7548e 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -92,7 +92,7 @@ For a more detailed explanation of how the test proxy works, along with links to ### Via Local Compile or .NET -1. [Install .Net 5.0 or 6.0](https://dotnet.microsoft.com/download) +1. [Install .NET 5.0 or 6.0](https://dotnet.microsoft.com/download) 2. Install test-proxy ```powershell diff --git a/tools/test-proxy/CONTRIBUTING.md b/tools/test-proxy/CONTRIBUTING.md index cabd5b672ea..5a0c0285296 100644 --- a/tools/test-proxy/CONTRIBUTING.md +++ b/tools/test-proxy/CONTRIBUTING.md @@ -2,7 +2,7 @@ Within this folder are all the components that make up the the tool `Azure.Sdk.Tools.TestProxy`, henceforth referred to as the `test-proxy`. -This product is currently developed on Visual Studio 2022, but CLI build/test also work fine. Currently, this project only supports `.net 6`, though `.net 7` support is coming. +This product is currently developed on Visual Studio 2022, but CLI build/test also work fine. Currently, this project only supports .NET 6.0, though .NET 7.0 support is coming. To contribute a change: From f05a61d1413c28937653f23646ee2ea7f55b4061 Mon Sep 17 00:00:00 2001 From: Pan Shao <97225342+pshao25@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:51:19 +0800 Subject: [PATCH 41/93] Add rules for convenience method and protocol method (#6934) * Add rules for convenience method and protocol method * update * updadte --- .../AZC0004Tests.cs | 34 +- .../AZC0017Tests.cs | 64 +++ .../AZC0018Tests.cs | 367 ++++++++++++++++++ .../AZC0019Tests.cs | 51 +++ .../ClientMethodsAnalyzer.cs | 164 +++++++- .../Azure.ClientSdk.Analyzers/Descriptors.cs | 18 + 6 files changed, 687 insertions(+), 11 deletions(-) create mode 100644 src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0017Tests.cs create mode 100644 src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0018Tests.cs create mode 100644 src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0019Tests.cs diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0004Tests.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0004Tests.cs index ca9cddf34a5..d048a9e7f62 100644 --- a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0004Tests.cs +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0004Tests.cs @@ -364,5 +364,37 @@ await Verifier.CreateAnalyzer(code) .WithDisabledDiagnostics("AZC0015") .RunAsync(); } + + [Fact] + public async Task AZC0004NotProducedForMethodsWithOverloadAlternative() + { + const string code = @" +using Azure; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(CancellationToken cancellationToken = default) + { + return null; + } + + public virtual Response Get() + { + return null; + } + + public virtual Response Get(CancellationToken cancellationToken) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } } -} \ No newline at end of file +} diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0017Tests.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0017Tests.cs new file mode 100644 index 00000000000..e4b33099ec5 --- /dev/null +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0017Tests.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Xunit; +using Verifier = Azure.ClientSdk.Analyzers.Tests.AzureAnalyzerVerifier; + +namespace Azure.ClientSdk.Analyzers.Tests +{ + public class AZC0017Tests + { + [Fact] + public async Task AZC0017ProducedForMethodsWithRequestContentParameter() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task {|AZC0017:GetAsync|}(RequestContent content, CancellationToken cancellationToken = default) + { + return null; + } + public virtual Response {|AZC0017:Get|}(RequestContent content, CancellationToken cancellationToken = default) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0017NotProducedForMethodsWithCancellationToken() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(string s, CancellationToken cancellationToken = default) + { + return null; + } + public virtual Response Get(string s, CancellationToken cancellationToken = default) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + } +} diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0018Tests.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0018Tests.cs new file mode 100644 index 00000000000..f4bf3e5c723 --- /dev/null +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0018Tests.cs @@ -0,0 +1,367 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Xunit; +using Verifier = Azure.ClientSdk.Analyzers.Tests.AzureAnalyzerVerifier; + +namespace Azure.ClientSdk.Analyzers.Tests +{ + public class AZC0018Tests + { + [Fact] + public async Task AZC0018NotProducedForCorrectReturnType() + { + const string code = @" +using Azure; +using Azure.Core; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task> GetHeadAsBooleanAsync(string s, RequestContext context) + { + return null; + } + + public virtual Response GetHeadAsBoolean(string s, RequestContext context) + { + return null; + } + + public virtual Task GetResponseAsync(string s, RequestContext context) + { + return null; + } + + public virtual Response GetResponse(string s, RequestContext context) + { + return null; + } + + public virtual AsyncPageable GetPageableAsync(string s, RequestContext context) + { + return null; + } + + public virtual Pageable GetPageable(string s, RequestContext context) + { + return null; + } + + public virtual Task GetOperationAsync(string s, RequestContext context) + { + return null; + } + + public virtual Operation GetOperation(string s, RequestContext context) + { + return null; + } + + public virtual Task> GetOperationOfTAsync(string s, RequestContext context) + { + return null; + } + + public virtual Operation GetOperationOfT(string s, RequestContext context) + { + return null; + } + + public virtual Task>> GetOperationOfPageableAsync(string s, RequestContext context) + { + return null; + } + + public virtual Operation> GetOperationOfPageable(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithGenericResponseOfPrimitive() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task> {|AZC0018:GetAsync|}(string s, RequestContext context) + { + return null; + } + + public virtual Response {|AZC0018:Get|}(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithGenericResponseOfModel() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class Model + { + string a; + } + public class SomeClient + { + public virtual Task> {|AZC0018:GetAsync|}(string s, RequestContext context) + { + return null; + } + + public virtual Response {|AZC0018:Get|}(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithPageableOfModel() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class Model + { + string a; + } + public class SomeClient + { + public virtual AsyncPageable {|AZC0018:GetAsync|}(string s, RequestContext context) + { + return null; + } + + public virtual Pageable {|AZC0018:Get|}(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithOperationOfModel() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class Model + { + string a; + } + public class SomeClient + { + public virtual Task> {|AZC0018:GetAsync|}(string s, RequestContext context) + { + return null; + } + + public virtual Operation {|AZC0018:Get|}(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithOperationOfPageableModel() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; + +namespace RandomNamespace +{ + public class Model + { + string a; + } + public class SomeClient + { + public virtual Task>> {|AZC0018:GetAsync|}(string s, RequestContext context) + { + return null; + } + + public virtual Operation> {|AZC0018:Get|}(string s, RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018ProducedForMethodsWithParameterModel() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +namespace RandomNamespace +{ + public struct Model + { + string a; + } + public class SomeClient + { + public virtual Task {|AZC0018:GetAsync|}(Model model, Azure.RequestContext context) + { + return null; + } + + public virtual Response {|AZC0018:Get|}(Model model, Azure.RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018NotProducedForMethodsWithNoRequestContentAndRequiredContext() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(string a, Azure.RequestContext context) + { + return null; + } + + public virtual Response Get(string a, Azure.RequestContext context) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018NotProducedForMethodsWithNoRequestContentButOnlyProtocol() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(string a, Azure.RequestContext context = null) + { + return null; + } + + public virtual Response Get(string a, Azure.RequestContext context = null) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + + [Fact] + public async Task AZC0018NotProducedForMethodsWithRequestContentAndOptionalRequestContext() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(RequestContent content, RequestContext context = null) + { + return null; + } + + public virtual Response Get(RequestContent content, RequestContext context = null) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + } +} diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0019Tests.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0019Tests.cs new file mode 100644 index 00000000000..c6626adf639 --- /dev/null +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers.Tests/AZC0019Tests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Xunit; +using Verifier = Azure.ClientSdk.Analyzers.Tests.AzureAnalyzerVerifier; + +namespace Azure.ClientSdk.Analyzers.Tests +{ + public class AZC0019Tests + { + [Fact] + public async Task AZC0019ProducedForMethodsWithNoRequestContentButProtocolAndConvenience() + { + const string code = @" +using Azure; +using Azure.Core; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +namespace RandomNamespace +{ + public class SomeClient + { + public virtual Task GetAsync(string a, CancellationToken cancellationToken = default) + { + return null; + } + + public virtual Response Get(string a, CancellationToken cancellationToken = default) + { + return null; + } + + public virtual Task {|AZC0019:GetAsync|}(string a, Azure.RequestContext context = null) + { + return null; + } + + public virtual Response {|AZC0019:Get|}(string a, Azure.RequestContext context = null) + { + return null; + } + } +}"; + await Verifier.CreateAnalyzer(code) + .RunAsync(); + } + } +} diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/ClientMethodsAnalyzer.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/ClientMethodsAnalyzer.cs index d771f2404cd..5999d35d71b 100644 --- a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/ClientMethodsAnalyzer.cs +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/ClientMethodsAnalyzer.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using Microsoft.CodeAnalysis; @@ -15,19 +16,29 @@ public class ClientMethodsAnalyzer : ClientAnalyzerBase private const string PageableTypeName = "Pageable"; private const string AsyncPageableTypeName = "AsyncPageable"; + private const string BinaryDataTypeName = "BinaryData"; private const string ResponseTypeName = "Response"; private const string NullableResponseTypeName = "NullableResponse"; private const string OperationTypeName = "Operation"; private const string TaskTypeName = "Task"; + private const string BooleanTypeName = "Boolean"; public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(new[] { Descriptors.AZC0002, Descriptors.AZC0003, Descriptors.AZC0004, - Descriptors.AZC0015 + Descriptors.AZC0015, + Descriptors.AZC0017, + Descriptors.AZC0018, + Descriptors.AZC0019, }); + private static bool IsRequestContent(IParameterSymbol parameterSymbol) + { + return parameterSymbol.Type.Name == "RequestContent"; + } + private static bool IsRequestContext(IParameterSymbol parameterSymbol) { return parameterSymbol.Name == "context" && parameterSymbol.Type.Name == "RequestContext"; @@ -43,7 +54,7 @@ private static bool IsCancellationOrRequestContext(IParameterSymbol parameterSym return IsCancellationToken(parameterSymbol) || IsRequestContext(parameterSymbol); } - private static void CheckIsLastArgumentCancellationTokenOrRequestContext(ISymbolAnalysisContext context, IMethodSymbol member) + private static void CheckServiceMethod(ISymbolAnalysisContext context, IMethodSymbol member) { var lastArgument = member.Parameters.LastOrDefault(); var isLastArgumentCancellationOrRequestContext = lastArgument != null && IsCancellationOrRequestContext(lastArgument); @@ -61,18 +72,151 @@ private static void CheckIsLastArgumentCancellationTokenOrRequestContext(ISymbol context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0002, member.Locations.FirstOrDefault()), member); } } - else if (IsCancellationToken(lastArgument) && !lastArgument.IsOptional) + else if (IsCancellationToken(lastArgument)) { - var overloadWithCancellationToken = FindMethod( - member.ContainingType.GetMembers(member.Name).OfType(), - member.TypeParameters, - member.Parameters.RemoveAt(member.Parameters.Length - 1)); + if (!lastArgument.IsOptional) + { + var overloadWithCancellationToken = FindMethod( + member.ContainingType.GetMembers(member.Name).OfType(), + member.TypeParameters, + member.Parameters.RemoveAt(member.Parameters.Length - 1)); + + if (overloadWithCancellationToken == null) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0002, member.Locations.FirstOrDefault()), member); + } + } - if (overloadWithCancellationToken == null) + if (member.Parameters.FirstOrDefault(IsRequestContent) != null) { - context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0002, member.Locations.FirstOrDefault()), member); + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0017, member.Locations.FirstOrDefault()), member); + } + } + else if (IsRequestContext(lastArgument)) + { + CheckProtocolMethodReturnType(context, member); + CheckProtocolMethodParameters(context, member); + } + } + + private static string GetFullNamespaceName(IParameterSymbol parameter) + { + var currentNamespace = parameter.Type.ContainingNamespace; + string currentName = currentNamespace.Name; + string fullNamespace = ""; + while (!string.IsNullOrEmpty(currentName)) + { + fullNamespace = fullNamespace == "" ? currentName : $"{currentName}.{fullNamespace}"; + currentNamespace = currentNamespace.ContainingNamespace; + currentName = currentNamespace.Name; + } + return fullNamespace; + } + + // A protocol method should not have model as parameter. If it has ambiguity with convenience method, it should have required RequestContext. + // Ambiguity: doesn't have a RequestContent, but there is a method ending with CancellationToken has same type of parameters + // No ambiguity: has RequestContent. + private static void CheckProtocolMethodParameters(ISymbolAnalysisContext context, IMethodSymbol method) + { + var containsModel = method.Parameters.Any(p => + { + var fullNamespace = GetFullNamespaceName(p); + return !fullNamespace.StartsWith("System") && !fullNamespace.StartsWith("Azure"); + }); + + if (containsModel) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); + return; + } + + var requestContent = method.Parameters.FirstOrDefault(IsRequestContent); + if (requestContent == null && method.Parameters.Last().IsOptional) + { + INamedTypeSymbol type = (INamedTypeSymbol)context.Symbol; + IEnumerable methodList = type.GetMembers(method.Name).OfType().Where(member => !SymbolEqualityComparer.Default.Equals(member, method)); + ImmutableArray parametersWithoutLast = method.Parameters.RemoveAt(method.Parameters.Length - 1); + IMethodSymbol convenienceMethod = FindMethod(methodList, method.TypeParameters, parametersWithoutLast, symbol => IsCancellationToken(symbol)); + if (convenienceMethod != null) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0019, method.Locations.FirstOrDefault()), method); + } + } + } + + // A protocol method should not have model as type. Accepted return type: Response, Task, Response, Task>, Pageable, AsyncPageable, Operation, Task>, Operation, Task, Operation>, Task>> + private static void CheckProtocolMethodReturnType(ISymbolAnalysisContext context, IMethodSymbol method) + { + bool IsValidPageable(ITypeSymbol typeSymbol) + { + var pageableTypeSymbol = typeSymbol as INamedTypeSymbol; + if (!pageableTypeSymbol.IsGenericType) + { + return false; + } + + var pageableReturn = pageableTypeSymbol.TypeArguments.Single(); + if (!IsOrImplements(pageableReturn, BinaryDataTypeName)) + { + return false; } + + return true; + } + + ITypeSymbol originalType = method.ReturnType; + ITypeSymbol unwrappedType = method.ReturnType; + + if (method.ReturnType is INamedTypeSymbol namedTypeSymbol && + namedTypeSymbol.IsGenericType && + namedTypeSymbol.Name == TaskTypeName) + { + unwrappedType = namedTypeSymbol.TypeArguments.Single(); } + + if (IsOrImplements(unwrappedType, ResponseTypeName)) + { + if (unwrappedType is INamedTypeSymbol responseTypeSymbol && responseTypeSymbol.IsGenericType) + { + var responseReturn = responseTypeSymbol.TypeArguments.Single(); + if (responseReturn.Name != BooleanTypeName) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); + } + } + return; + } + else if (IsOrImplements(unwrappedType, OperationTypeName)) + { + if (unwrappedType is INamedTypeSymbol operationTypeSymbol && operationTypeSymbol.IsGenericType) + { + var operationReturn = operationTypeSymbol.TypeArguments.Single(); + if (IsOrImplements(operationReturn, PageableTypeName) || IsOrImplements(operationReturn, AsyncPageableTypeName)) + { + if (!IsValidPageable(operationReturn)) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); + } + return; + } + + if (!IsOrImplements(operationReturn, BinaryDataTypeName)) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); + } + } + return; + } + else if (IsOrImplements(originalType, PageableTypeName) || IsOrImplements(originalType, AsyncPageableTypeName)) + { + if (!IsValidPageable(originalType)) + { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); + } + return; + } + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.AZC0018, method.Locations.FirstOrDefault()), method); } private static void CheckClientMethod(ISymbolAnalysisContext context, IMethodSymbol member) @@ -164,7 +308,7 @@ public override void AnalyzeCore(ISymbolAnalysisContext context) if (IsClientMethodReturnType(context, methodSymbol, false)) { - CheckIsLastArgumentCancellationTokenOrRequestContext(context, methodSymbol); + CheckServiceMethod(context, methodSymbol); } } } diff --git a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/Descriptors.cs b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/Descriptors.cs index 0b292863d5b..87380df0a26 100644 --- a/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/Descriptors.cs +++ b/src/dotnet/Azure.ClientSdk.Analyzers/Azure.ClientSdk.Analyzers/Descriptors.cs @@ -112,6 +112,24 @@ internal class Descriptors "Usage", DiagnosticSeverity.Warning, true); + public static DiagnosticDescriptor AZC0017 = new DiagnosticDescriptor( + nameof(AZC0017), + "Invalid convenience method signature.", + "Convenience methods shouldn't have parameters with the RequestContent type.", + "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: null); + + public static DiagnosticDescriptor AZC0018 = new DiagnosticDescriptor( + nameof(AZC0018), + "Invalid protocol method signature.", + "Protocol methods should take a RequestContext parameter called `context` and not use a model type in a parameter or return type.", + "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: null); + + public static DiagnosticDescriptor AZC0019 = new DiagnosticDescriptor( + nameof(AZC0019), + "Potential ambiguous call exists.", + "There will be an ambiguous call error when the user calls with only the required parameters. All parameters of the protocol method should be required.", + "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: null); + public static DiagnosticDescriptor AZC0020 = new DiagnosticDescriptor( nameof(AZC0020), "Avoid using banned types in public APIs", From 4cf46535168a2df740d49f61a33b3d9aed0a5c4f Mon Sep 17 00:00:00 2001 From: Libba Lawrence Date: Thu, 28 Sep 2023 10:16:23 -0700 Subject: [PATCH 42/93] [Pylint] missing kw only args (#7006) * missing kw only args * update tests * also on class * Update tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py Co-authored-by: Krista Pratico * update comparison test * update naming * update readme * order of param --------- Co-authored-by: Krista Pratico --- .../CHANGELOG.md | 4 + .../azure-pylint-guidelines-checker/README.md | 1 + .../pylint_guidelines_checker.py | 82 +++++++++++++++---- .../azure-pylint-guidelines-checker/setup.py | 2 +- .../tests/test_pylint_custom_plugins.py | 28 ++++++- 5 files changed, 97 insertions(+), 20 deletions(-) diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md index 893571419a9..d0cce88a725 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## 0.2.0 (Unreleased) + +- Checker to enforce docstring keywords being keyword-only in method signature. + ## 0.1.0 (2023-08-02) Add two new checkers: diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md b/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md index 15d60503aa4..fb9e5caf308 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md @@ -89,3 +89,4 @@ docstring-missing-rtype | pylint:disable=docstring-missing-rtype | Docs docstring-should-be-keyword | pylint:disable=docstring-should-be-keyword | Docstring should use keywords. | [link](https://azure.github.io/azure-sdk/python_documentation.html#docstrings) | do-not-import-legacy-six | pylint:disable=do-not-import-legacy-six | Do not import six. | No Link. | no-legacy-azure-core-http-response-import | pylint:disable=no-legacy-azure-core-http-response-import | Do not import HttpResponse from azure.core.pipeline.transport outside of Azure Core. You can import HttpResponse from azure.core.rest instead. | [link](https://github.com/Azure/azure-sdk-for-python/issues/30785) | +docstring-keyword-should-match-keyword-only | pylint:disable=docstring-keyword-should-match-keyword-only | Docstring keyword arguments and keyword-only method arguments should match. | [link](https://azure.github.io/azure-sdk/python_documentation.html#docstrings) | diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py index 53f093a532e..8dd0223f2ff 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py @@ -4,7 +4,7 @@ # ------------------------------------ """ -Pylint custom checkers for SDK guidelines: C4717 - C4749 +Pylint custom checkers for SDK guidelines: C4717 - C4758 """ import logging @@ -1228,6 +1228,12 @@ class CheckDocstringParameters(BaseChecker): "docstring-should-be-keyword", "Docstring should use keywords.", ), + "C4758": ( + '"%s" missing in docstring or in method signature. There should be a direct correlation between :keyword: arguments in the docstring and keyword-only arguments in method signature. See details: ' + 'https://azure.github.io/azure-sdk/python_documentation.html#docstrings', + "docstring-keyword-should-match-keyword-only", + "Docstring keyword arguments and keyword-only method arguments should match.", + ), } options = ( ( @@ -1280,6 +1286,45 @@ class CheckDocstringParameters(BaseChecker): def __init__(self, linter=None): super(CheckDocstringParameters, self).__init__(linter) + def _find_keyword(self, line): + keyword_args = {} + # this param has its type on a separate line + if line.startswith("keyword") and line.count(" ") == 1: + param = line.split("keyword ")[1] + keyword_args[param] = None + # this param has its type on the same line + if line.startswith("keyword") and line.count(" ") == 2: + _, param_type, param = line.split(" ") + keyword_args[param] = param_type + # if the param has its type on the same line with additional spaces + if line.startswith("keyword") and line.count(" ") > 2: + param = line.split(" ")[-1] + param_type = ("").join(line.split(" ")[1:-1]) + keyword_args[param] = param_type + + return keyword_args + + def _find_param(self, line, docstring, idx, docparams): + # this param has its type on a separate line + if line.startswith("param") and line.count(" ") == 1: + param = line.split("param ")[1] + docparams[param] = None + # this param has its type on the same line + if line.startswith("param") and line.count(" ") == 2: + _, param_type, param = line.split(" ") + docparams[param] = param_type + # if the param has its type on the same line with additional spaces + if line.startswith("param") and line.count(" ") > 2: + param = line.split(" ")[-1] + param_type = ("").join(line.split(" ")[1:-1]) + docparams[param] = param_type + if line.startswith("type"): + param = line.split("type ")[1] + if param in docparams: + docparams[param] = docstring[idx+1] + + return docparams + def check_parameters(self, node): """Parse the docstring for any params and types and compares it to the function's parameters. @@ -1290,22 +1335,26 @@ def check_parameters(self, node): 3. Missing a return doc in the docstring when a function returns something. 4. Missing an rtype in the docstring when a function returns something. 5. Extra params in docstring that aren't function parameters. Change to keywords. + 6. Docstring has a keyword that isn't a keyword-only argument in the function signature. :param node: ast.ClassDef or ast.FunctionDef :return: None """ arg_names = [] + method_keyword_only_args = [] vararg_name = None # specific case for constructor where docstring found in class def if isinstance(node, astroid.ClassDef): for constructor in node.body: if isinstance(constructor, astroid.FunctionDef) and constructor.name == "__init__": arg_names = [arg.name for arg in constructor.args.args] + method_keyword_only_args = [arg.name for arg in constructor.args.kwonlyargs] vararg_name = node.args.vararg break if isinstance(node, astroid.FunctionDef): arg_names = [arg.name for arg in node.args.args] + method_keyword_only_args = [arg.name for arg in node.args.kwonlyargs] vararg_name = node.args.vararg try: @@ -1319,25 +1368,14 @@ def check_parameters(self, node): arg_names.append(vararg_name) docparams = {} + docstring_keyword_args = {} for idx, line in enumerate(docstring): - # this param has its type on a separate line - if line.startswith("param") and line.count(" ") == 1: - param = line.split("param ")[1] - docparams[param] = None - # this param has its type on the same line - if line.startswith("param") and line.count(" ") == 2: - _, param_type, param = line.split(" ") - docparams[param] = param_type - # if the param has its type on the same line with additional spaces - if line.startswith("param") and line.count(" ") > 2: - param = line.split(" ")[-1] - param_type = ("").join(line.split(" ")[1:-1]) - docparams[param] = param_type - if line.startswith("type"): - param = line.split("type ")[1] - if param in docparams: - docparams[param] = docstring[idx+1] + # check for keyword args in docstring + docstring_keyword_args.update(self._find_keyword(line)) + # check for params in docstring + docparams.update(self._find_param(line, docstring, idx, docparams)) + # check that all params are documented missing_params = [] for param in arg_names: @@ -1346,11 +1384,19 @@ def check_parameters(self, node): if param not in docparams: missing_params.append(param) + # check that all keyword-only args are documented + missing_kwonly_args = list(set(docstring_keyword_args) ^ set(method_keyword_only_args)) + if missing_params: self.add_message( msgid="docstring-missing-param", args=(", ".join(missing_params)), node=node, confidence=None ) + if missing_kwonly_args: + self.add_message( + msgid="docstring-keyword-should-match-keyword-only", args=(", ".join(missing_kwonly_args)), node=node, confidence=None + ) + # check if we have a type for each param and check if documented params that should be keywords missing_types = [] should_be_keywords = [] diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/setup.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/setup.py index 435487173f9..866d5835434 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/setup.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/setup.py @@ -6,7 +6,7 @@ setup( name="azure-pylint-guidelines-checker", - version="0.1.0", + version="0.2.0", url='http://github.com/Azure/azure-sdk-for-python', license='MIT License', description="A pylint plugin which enforces azure sdk guidelines.", diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py index 9e26a9567d7..99c3a35a69c 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py @@ -3422,6 +3422,15 @@ def function_foo(*x, y, z): """ ) with self.assertAddsMessages( + pylint.testutils.MessageTest( + msg_id='docstring-keyword-should-match-keyword-only', + line=2, + node=node, + args='z, y', + col_offset=0, + end_line=2, + end_col_offset=16 + ), pylint.testutils.MessageTest( msg_id="docstring-missing-type", line=2, @@ -3439,7 +3448,7 @@ def function_foo(*x, y, z): col_offset=0, end_line=2, end_col_offset=16 - ) + ), ): self.checker.visit_functiondef(node) @@ -3523,6 +3532,23 @@ def function_foo(): with self.assertNoMessages(): self.checker.visit_functiondef(node) + def test_docstring_keyword_only(self): + node = astroid.extract_node( + """ + def function_foo(self, x, *, z, y=None): + ''' + :param x: x + :type x: str + :keyword str y: y + :keyword str z: z + ''' + print("hello") + """ + ) + with self.assertNoMessages(): + self.checker.visit_functiondef(node) + + class TestDoNotImportLegacySix(pylint.testutils.CheckerTestCase): """Test that we are blocking disallowed imports and allowing allowed imports.""" CHECKER_CLASS = checker.DoNotImportLegacySix From bb2e2596f11cc9cd814c63ebd2ce7fbbd3ec9cb2 Mon Sep 17 00:00:00 2001 From: Srikanta <51379715+srnagar@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:11:38 -0700 Subject: [PATCH 43/93] Add KeyCredential diagnostics (#7007) --- .../azure/tools/apiview/processor/diagnostics/Diagnostics.java | 2 +- .../diagnostics/rules/BuilderTraitsDiagnosticRule.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/Diagnostics.java b/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/Diagnostics.java index 35db8e07745..a5d787fc033 100644 --- a/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/Diagnostics.java +++ b/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/Diagnostics.java @@ -60,7 +60,7 @@ public Diagnostics() { .add("clientOptions", new ExactTypeNameCheckFunction("ClientOptions")) .add("connectionString", new ExactTypeNameCheckFunction("String")) .add("credential", new ExactTypeNameCheckFunction(new ParameterAllowedTypes("TokenCredential", - "AzureKeyCredential", "AzureSasCredential", "AzureNamedKeyCredential"))) + "AzureKeyCredential", "AzureSasCredential", "AzureNamedKeyCredential", "KeyCredential"))) .add("endpoint", new ExactTypeNameCheckFunction("String")) .add("serviceVersion", this::checkServiceVersionType)); diagnostics.add(new RequiredBuilderMethodsDiagnosticRule("amqp") diff --git a/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/rules/BuilderTraitsDiagnosticRule.java b/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/rules/BuilderTraitsDiagnosticRule.java index 0cbc8977408..bdbbea304e9 100644 --- a/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/rules/BuilderTraitsDiagnosticRule.java +++ b/src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/diagnostics/rules/BuilderTraitsDiagnosticRule.java @@ -28,6 +28,9 @@ public class BuilderTraitsDiagnosticRule implements DiagnosticRule { public BuilderTraitsDiagnosticRule() { traits = new HashMap<>(); + traits.put("KeyCredentialTrait", new TraitClass( + new TraitMethod("credential", "KeyCredential") + )); traits.put("AzureKeyCredentialTrait", new TraitClass( new TraitMethod("credential", "AzureKeyCredential") )); From 370f3dfd9272275ce0022b4e1b255c4c5a266640 Mon Sep 17 00:00:00 2001 From: Jesse Squire Date: Fri, 29 Sep 2023 09:32:40 -0700 Subject: [PATCH 44/93] [JimBot] Remove CXP Attention rule (#7033) * [JimBot] Remove CXP Attention rule The focus of these changes is to remove the rule attached to the "CXP Attention" label that has been removed from the Azure SDK repositories. Also riding along is a tweak to the "needs-team-triage" criteria to avoid applying it when an issue has already been assigned. --- .../Static/IssueProcessingTests.cs | 102 ++----- .../CXPAttention_issue_labeled.json | 275 ----------------- ...n_issue_labeled_has_service-attention.json | 284 ------------------ ...ssedReset_issue_labeled_CXP_attention.json | 275 ----------------- ...d_CXP_attention_has_service_attention.json | 284 ------------------ ...d_service_attention_has_CXP_attention.json | 284 ------------------ ...beled_service_attention_no_assignees.json} | 56 +--- .../Constants/LabelConstants.cs | 1 - .../EventProcessing/IssueProcessing.cs | 59 +--- tools/github-event-processor/RULES.md | 29 +- .../YmlAndConfigFiles/event-processor.config | 1 - tools/github/data/common-labels.csv | 1 - 12 files changed, 46 insertions(+), 1605 deletions(-) delete mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled.json delete mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled_has_service-attention.json delete mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json delete mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention_has_service_attention.json delete mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_has_CXP_attention.json rename tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/{ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json => ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_no_assignees.json} (81%) diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Static/IssueProcessingTests.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Static/IssueProcessingTests.cs index 5fa838548d6..5b27c8732a1 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Static/IssueProcessingTests.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Static/IssueProcessingTests.cs @@ -23,7 +23,7 @@ public class IssueProcessingTests : ProcessingTestBase /// Conditions: Issue has no assignee /// Issue has no labels /// Resulting Action:Query the AI Service - /// If the AI Service + /// If the AI Service /// /// String, RulesConstants for the rule being tested /// JSon payload file for the event being tested @@ -59,7 +59,7 @@ public async Task TestInitialIssueTriage(string rule, string payloadFile, RuleSt } await IssueProcessing.InitialIssueTriage(mockGitHubEventClient, issueEventPayload); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); if (RuleState.On == ruleState) { @@ -101,8 +101,8 @@ public async Task TestInitialIssueTriage(string rule, string payloadFile, RuleSt } /// - /// ManualIssueTriage requires 2 labeled event payloads to test. - /// 1. Labeled payload needs needs to be one where needs-triage is being added which should cause no change. + /// ManualIssueTriage requires 2 labeled event payloads to test. + /// 1. Labeled payload needs needs to be one where needs-triage is being added which should cause no change. /// 2. Needs-triage needs to be already on the issues when another label is added. This should cause an update /// where the needs-triage label is removed. This is also the payload used to check when the rule is Off. /// Trigger: issues labeled @@ -127,7 +127,7 @@ public async Task TestManualIssueTriage(string rule, string payloadFile, RuleSta var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.ManualIssueTriage(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -191,7 +191,7 @@ public async Task TestServiceAttention(string rule, string payloadFile, RuleStat } IssueProcessing.ServiceAttention(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -220,66 +220,14 @@ public async Task TestServiceAttention(string rule, string payloadFile, RuleStat } } - - /// - /// Test CXPAttention rule enabled/disabled, with a payload that would cause updates when enabled. - /// Verify all the expected updates when enabled and no updates when disabled. - /// Trigger: issues labeled - /// Conditions: Issue is open - /// Label being added is "CXP-Attention" - /// Does not have "Service-Attention" label - /// Resulting Action: Add issues comment "Thank you for your feedback. This has been routed to the support team for assistance." - /// - /// String, RulesConstants for the rule being tested - /// JSon payload file for the event being tested - /// Whether or not the rule is on/off - /// Where or not the payload has the Service Attention label - [Category("static")] - [TestCase(RulesConstants.CXPAttention, "Tests.JsonEventPayloads/CXPAttention_issue_labeled.json", RuleState.Off, false)] - [TestCase(RulesConstants.CXPAttention, "Tests.JsonEventPayloads/CXPAttention_issue_labeled.json", RuleState.On, false)] - [TestCase(RulesConstants.CXPAttention, "Tests.JsonEventPayloads/CXPAttention_issue_labeled_has_service-attention.json", RuleState.Off, true)] - - public async Task TestCXPAttention(string rule, string payloadFile, RuleState ruleState, bool hasServiceAttentionLabel) - { - var mockGitHubEventClient = new MockGitHubEventClient(OrgConstants.ProductHeaderName); - mockGitHubEventClient.RulesConfiguration.Rules[rule] = ruleState; - var rawJson = TestHelpers.GetTestEventPayload(payloadFile); - var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); - IssueProcessing.CXPAttention(mockGitHubEventClient, issueEventPayload); - - // Verify the RuleCheck - Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); - - var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); - if (RuleState.On == ruleState) - { - // If the issues has the ServiceAttention label then there should be no updates - if (hasServiceAttentionLabel) - { - Assert.AreEqual(0, totalUpdates, $"Issue has {LabelConstants.ServiceAttention} and should not have produced any updates."); - } - else - { - // There should be one comment created and no other updates - Assert.AreEqual(1, totalUpdates, $"The number of updates should have been 1 but was instead, {totalUpdates}"); - // Verify that a single comment was created - Assert.AreEqual(1, mockGitHubEventClient.GetComments().Count, $"{rule} should have produced a single comment."); - } - } - else - { - Assert.AreEqual(0, totalUpdates, $"{rule} is {ruleState} and should not have produced any updates."); - } - } - /// /// Test ManualTriageAfterExternalAssignment rule enabled/disabled, with a payload that would cause updates when enabled. /// Verify all the expected updates when enabled and no updates when disabled. /// Trigger: issue unlabeled /// Conditions: Issue is open /// Has "customer-reported" label - /// Label removed is "Service Attention" OR "CXP Attention" - /// Issue does not have "Service Attention" OR "CXP Attention" + /// Issue is unassigned + /// Label removed is "Service Attention" /// (in other words if both labels are on the issue and one is removed, this /// shouldn't process) /// Resulting Action: Add "needs-team-triage" label @@ -289,12 +237,9 @@ public async Task TestCXPAttention(string rule, string payloadFile, RuleState ru /// Whether or not the rule is on/off /// Whether or not the payload already has the needs-team-triage label [Category("static")] - [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json", RuleState.Off, false, false)] - [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json", RuleState.On, false, true)] - [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention_has_service_attention.json", RuleState.On, false, false)] [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention.json", RuleState.Off, false, false)] - [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention.json", RuleState.On, false, true)] - [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_has_CXP_attention.json", RuleState.On, false, false)] + [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention.json", RuleState.On, false, false)] + [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_no_assignees.json", RuleState.On, false, true)] [TestCase(RulesConstants.ManualTriageAfterExternalAssignment, "Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_has_needs-team-triage.json", RuleState.On, true, false)] public async Task TestManualTriageAfterExternalAssignment(string rule, string payloadFile, RuleState ruleState, bool alreadyHasNeedsTeamTriage, bool shouldAddLabel) { @@ -304,7 +249,7 @@ public async Task TestManualTriageAfterExternalAssignment(string rule, string pa var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.ManualTriageAfterExternalAssignment(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -326,7 +271,7 @@ public async Task TestManualTriageAfterExternalAssignment(string rule, string pa } else { - Assert.AreEqual(0, totalUpdates, $"The issue only 1 of {LabelConstants.CXPAttention} or {LabelConstants.ServiceAttention}. With the other still being on the issue there should have been no updates."); + Assert.AreEqual(0, totalUpdates, $"The issue only 1 of {LabelConstants.ServiceAttention}. With the other still being on the issue there should have been no updates."); } } } @@ -342,7 +287,7 @@ public async Task TestManualTriageAfterExternalAssignment(string rule, string pa /// Trigger: issue reopened/edited /// Conditions: Issue is open OR Issue is being reopened /// Issue has "no-recent-activity" label - /// User modifying the issue is NOT a known bot + /// User modifying the issue is NOT a known bot /// /// String, RulesConstants for the rule being tested /// JSon payload file for the event being tested @@ -360,7 +305,7 @@ public async Task TestResetIssueActivity(string rule, string payloadFile, RuleSt var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.ResetIssueActivity(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -408,7 +353,7 @@ public async Task TestRequireAttentionForNonMilestone(string rule, string payloa var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.RequireAttentionForNonMilestone(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -432,7 +377,7 @@ public async Task TestRequireAttentionForNonMilestone(string rule, string payloa /// Trigger: issue labeled /// Conditions: Issue is open /// Label added is "needs-author-feedback" - /// Resulting Action: + /// Resulting Action: /// Remove "needs-triage" label /// Remove "needs-team-triage" label /// Remove "needs-team-attention" label @@ -453,7 +398,7 @@ public async Task TestAuthorFeedbackNeeded(string rule, string payloadFile, Rule var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.AuthorFeedbackNeeded(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -492,7 +437,7 @@ public async Task TestAuthorFeedbackNeeded(string rule, string payloadFile, Rule /// Trigger: issue labeled /// Conditions: Issue is open /// Label added is "issue-addressed" - /// Resulting Action: + /// Resulting Action: /// Remove "needs-triage" label if it exists /// Remove "needs-team-triage" label /// Remove "needs-team-attention" label @@ -516,7 +461,7 @@ public async Task TestIssueAddressed(string rule, string payloadFile, RuleState var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.IssueAddressed(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); @@ -529,7 +474,7 @@ public async Task TestIssueAddressed(string rule, string payloadFile, RuleState } else { - // There should be one comment and up to 5 labels removed, the no-recent-activity and all needs-* labels removed + // There should be one comment and up to 5 labels removed, the no-recent-activity and all needs-* labels removed // (the test payload has them all but real events will only remove the ones that are there) Assert.AreEqual(6, totalUpdates, $"The number of updates should have been 2 but was instead, {totalUpdates}"); @@ -563,18 +508,15 @@ public async Task TestIssueAddressed(string rule, string payloadFile, RuleState /// "needs-team-attention" /// "needs-author-feedback" /// "Service Attention" - /// "CXP Attention" /// "needs-triage" /// "needs-team-triage" - /// Resulting Action: + /// Resulting Action: /// Remove "issue-addressed" label /// /// String, RulesConstants for the rule being tested /// JSon payload file for the event being tested /// Whether or not the rule is on/off [Category("static")] - [TestCase(RulesConstants.IssueAddressedReset, "Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json", RuleState.Off)] - [TestCase(RulesConstants.IssueAddressedReset, "Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json", RuleState.On)] [TestCase(RulesConstants.IssueAddressedReset, "Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_needs-author-feedack.json", RuleState.On)] [TestCase(RulesConstants.IssueAddressedReset, "Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_needs-team-attention.json", RuleState.On)] [TestCase(RulesConstants.IssueAddressedReset, "Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_needs-team-triage.json", RuleState.On)] @@ -588,7 +530,7 @@ public async Task TestIssueAddressedReset(string rule, string payloadFile, RuleS var issueEventPayload = SimpleJsonSerializer.Deserialize(rawJson); IssueProcessing.IssueAddressedReset(mockGitHubEventClient, issueEventPayload); - // Verify the RuleCheck + // Verify the RuleCheck Assert.AreEqual(ruleState == RuleState.On, mockGitHubEventClient.RulesConfiguration.RuleEnabled(rule), $"Rule '{rule}' enabled should have been {ruleState == RuleState.On} but RuleEnabled returned {ruleState != RuleState.On}.'"); var totalUpdates = await mockGitHubEventClient.ProcessPendingUpdates(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number); diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled.json deleted file mode 100644 index 7fbaeb2b22b..00000000000 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "action": "labeled", - "issue": { - "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], - "author_association": "OWNER", - "body": null, - "closed_at": null, - "comments": 0, - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/comments", - "created_at": "2023-01-27T17:01:30Z", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/events", - "html_url": "https://github.com/Azure/azure-sdk-fake/issues/14", - "id": 1560095682, - "labels": [ - { - "color": "d73a4a", - "default": true, - "description": "Something isn't working", - "id": 4273699693, - "name": "bug", - "node_id": "LA_kwDOHkcrQs7-u3tt", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/bug" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4704569627, - "name": "needs-triage", - "node_id": "LA_kwDOHkcrQs8AAAABGGoJGw", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/needs-triage" - }, - { - "color": "9C7082", - "default": false, - "description": "fake label for testing", - "id": 5095712487, - "name": "FakeLabel1", - "node_id": "LA_kwDOHkcrQs8AAAABL7pm5w", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/FakeLabel1" - }, - { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - } - ], - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/labels{/name}", - "locked": false, - "milestone": null, - "node_id": "I_kwDOHkcrQs5c_SvC", - "number": 14, - "performed_via_github_app": null, - "reactions": { - "+1": 0, - "-1": 0, - "confused": 0, - "eyes": 0, - "heart": 0, - "hooray": 0, - "laugh": 0, - "rocket": 0, - "total_count": 0, - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/reactions" - }, - "repository_url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "state": "open", - "state_reason": null, - "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", - "title": "New test issue to generate event payloads", - "updated_at": "2023-01-30T16:29:49Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", - "user": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - }, - "label": { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - }, - "repository": { - "allow_forking": true, - "archive_url": "https://api.github.com/repos/Azure/azure-sdk-fake/{archive_format}{/ref}", - "archived": false, - "assignees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/assignees{/user}", - "blobs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/Azure/azure-sdk-fake/branches{/branch}", - "clone_url": "https://github.com/Azure/azure-sdk-fake.git", - "collaborators_url": "https://api.github.com/repos/Azure/azure-sdk-fake/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/comments{/number}", - "commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/commits{/sha}", - "compare_url": "https://api.github.com/repos/Azure/azure-sdk-fake/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contents/{+path}", - "contributors_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contributors", - "created_at": "2022-06-27T16:19:29Z", - "default_branch": "main", - "deployments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/deployments", - "description": "Tools repository leveraged by the Azure SDK team.", - "disabled": false, - "downloads_url": "https://api.github.com/repos/Azure/azure-sdk-fake/downloads", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/events", - "fork": true, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/forks", - "full_name": "Azure/azure-sdk-fake", - "git_commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/tags{/sha}", - "git_url": "git://github.com/Azure/azure-sdk-fake.git", - "has_discussions": false, - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_projects": true, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/hooks", - "html_url": "https://github.com/Azure/azure-sdk-fake", - "id": 507980610, - "is_template": false, - "issue_comment_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/events{/number}", - "issues_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues{/number}", - "keys_url": "https://api.github.com/repos/Azure/azure-sdk-fake/keys{/key_id}", - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels{/name}", - "language": "C#", - "languages_url": "https://api.github.com/repos/Azure/azure-sdk-fake/languages", - "license": { - "key": "mit", - "name": "MIT License", - "node_id": "MDc6TGljZW5zZTEz", - "spdx_id": "MIT", - "url": "https://api.github.com/licenses/mit" - }, - "merges_url": "https://api.github.com/repos/Azure/azure-sdk-fake/merges", - "milestones_url": "https://api.github.com/repos/Azure/azure-sdk-fake/milestones{/number}", - "mirror_url": null, - "name": "azure-sdk-fake", - "node_id": "R_kgDOHkcrQg", - "notifications_url": "https://api.github.com/repos/Azure/azure-sdk-fake/notifications{?since,all,participating}", - "open_issues": 6, - "open_issues_count": 6, - "owner": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/Azure/azure-sdk-fake/pulls{/number}", - "pushed_at": "2023-01-27T16:33:00Z", - "releases_url": "https://api.github.com/repos/Azure/azure-sdk-fake/releases{/id}", - "size": 29098, - "ssh_url": "git@github.com:Azure/azure-sdk-fake.git", - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/stargazers", - "statuses_url": "https://api.github.com/repos/Azure/azure-sdk-fake/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscribers", - "subscription_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscription", - "svn_url": "https://github.com/Azure/azure-sdk-fake", - "tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/tags", - "teams_url": "https://api.github.com/repos/Azure/azure-sdk-fake/teams", - "topics": [], - "trees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/trees{/sha}", - "updated_at": "2023-01-23T19:54:18Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "visibility": "public", - "watchers": 0, - "watchers_count": 0, - "web_commit_signoff_required": false - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } -} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled_has_service-attention.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled_has_service-attention.json deleted file mode 100644 index 3024a04e768..00000000000 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/CXPAttention_issue_labeled_has_service-attention.json +++ /dev/null @@ -1,284 +0,0 @@ -{ - "action": "labeled", - "issue": { - "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], - "author_association": "OWNER", - "body": null, - "closed_at": null, - "comments": 0, - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/comments", - "created_at": "2023-01-27T17:01:30Z", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/events", - "html_url": "https://github.com/Azure/azure-sdk-fake/issues/14", - "id": 1560095682, - "labels": [ - { - "color": "d73a4a", - "default": true, - "description": "Something isn't working", - "id": 4273699693, - "name": "bug", - "node_id": "LA_kwDOHkcrQs7-u3tt", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/bug" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4704569627, - "name": "needs-triage", - "node_id": "LA_kwDOHkcrQs8AAAABGGoJGw", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/needs-triage" - }, - { - "color": "9C7082", - "default": false, - "description": "fake label for testing", - "id": 5095712487, - "name": "FakeLabel1", - "node_id": "LA_kwDOHkcrQs8AAAABL7pm5w", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/FakeLabel1" - }, - { - "color": "F75BB2", - "default": false, - "description": "", - "id": 5095715984, - "name": "Service Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7p0kA", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/Service%20Attention" - }, - { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - } - ], - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/labels{/name}", - "locked": false, - "milestone": null, - "node_id": "I_kwDOHkcrQs5c_SvC", - "number": 14, - "performed_via_github_app": null, - "reactions": { - "+1": 0, - "-1": 0, - "confused": 0, - "eyes": 0, - "heart": 0, - "hooray": 0, - "laugh": 0, - "rocket": 0, - "total_count": 0, - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/reactions" - }, - "repository_url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "state": "open", - "state_reason": null, - "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", - "title": "New test issue to generate event payloads", - "updated_at": "2023-01-30T16:26:17Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", - "user": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - }, - "label": { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - }, - "repository": { - "allow_forking": true, - "archive_url": "https://api.github.com/repos/Azure/azure-sdk-fake/{archive_format}{/ref}", - "archived": false, - "assignees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/assignees{/user}", - "blobs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/Azure/azure-sdk-fake/branches{/branch}", - "clone_url": "https://github.com/Azure/azure-sdk-fake.git", - "collaborators_url": "https://api.github.com/repos/Azure/azure-sdk-fake/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/comments{/number}", - "commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/commits{/sha}", - "compare_url": "https://api.github.com/repos/Azure/azure-sdk-fake/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contents/{+path}", - "contributors_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contributors", - "created_at": "2022-06-27T16:19:29Z", - "default_branch": "main", - "deployments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/deployments", - "description": "Tools repository leveraged by the Azure SDK team.", - "disabled": false, - "downloads_url": "https://api.github.com/repos/Azure/azure-sdk-fake/downloads", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/events", - "fork": true, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/forks", - "full_name": "Azure/azure-sdk-fake", - "git_commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/tags{/sha}", - "git_url": "git://github.com/Azure/azure-sdk-fake.git", - "has_discussions": false, - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_projects": true, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/hooks", - "html_url": "https://github.com/Azure/azure-sdk-fake", - "id": 507980610, - "is_template": false, - "issue_comment_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/events{/number}", - "issues_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues{/number}", - "keys_url": "https://api.github.com/repos/Azure/azure-sdk-fake/keys{/key_id}", - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels{/name}", - "language": "C#", - "languages_url": "https://api.github.com/repos/Azure/azure-sdk-fake/languages", - "license": { - "key": "mit", - "name": "MIT License", - "node_id": "MDc6TGljZW5zZTEz", - "spdx_id": "MIT", - "url": "https://api.github.com/licenses/mit" - }, - "merges_url": "https://api.github.com/repos/Azure/azure-sdk-fake/merges", - "milestones_url": "https://api.github.com/repos/Azure/azure-sdk-fake/milestones{/number}", - "mirror_url": null, - "name": "azure-sdk-fake", - "node_id": "R_kgDOHkcrQg", - "notifications_url": "https://api.github.com/repos/Azure/azure-sdk-fake/notifications{?since,all,participating}", - "open_issues": 6, - "open_issues_count": 6, - "owner": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/Azure/azure-sdk-fake/pulls{/number}", - "pushed_at": "2023-01-27T16:33:00Z", - "releases_url": "https://api.github.com/repos/Azure/azure-sdk-fake/releases{/id}", - "size": 29098, - "ssh_url": "git@github.com:Azure/azure-sdk-fake.git", - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/stargazers", - "statuses_url": "https://api.github.com/repos/Azure/azure-sdk-fake/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscribers", - "subscription_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscription", - "svn_url": "https://github.com/Azure/azure-sdk-fake", - "tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/tags", - "teams_url": "https://api.github.com/repos/Azure/azure-sdk-fake/teams", - "topics": [], - "trees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/trees{/sha}", - "updated_at": "2023-01-23T19:54:18Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "visibility": "public", - "watchers": 0, - "watchers_count": 0, - "web_commit_signoff_required": false - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } -} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json deleted file mode 100644 index 7df111c9ea4..00000000000 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/IssueAddressedReset_issue_labeled_CXP_attention.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "action": "labeled", - "issue": { - "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], - "author_association": "OWNER", - "body": null, - "closed_at": null, - "comments": 0, - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/comments", - "created_at": "2023-01-27T17:01:30Z", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/events", - "html_url": "https://github.com/Azure/azure-sdk-fake/issues/14", - "id": 1560095682, - "labels": [ - { - "color": "d73a4a", - "default": true, - "description": "Something isn't working", - "id": 4273699693, - "name": "bug", - "node_id": "LA_kwDOHkcrQs7-u3tt", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/bug" - }, - { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - }, - { - "color": "0679ED", - "default": false, - "description": "", - "id": 5074290839, - "name": "issue-addressed", - "node_id": "LA_kwDOHkcrQs8AAAABLnOIlw", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/issue-addressed" - }, - { - "color": "9C7082", - "default": false, - "description": "fake label for testing", - "id": 5095712487, - "name": "FakeLabel1", - "node_id": "LA_kwDOHkcrQs8AAAABL7pm5w", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/FakeLabel1" - } - ], - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/labels{/name}", - "locked": false, - "milestone": null, - "node_id": "I_kwDOHkcrQs5c_SvC", - "number": 14, - "performed_via_github_app": null, - "reactions": { - "+1": 0, - "-1": 0, - "confused": 0, - "eyes": 0, - "heart": 0, - "hooray": 0, - "laugh": 0, - "rocket": 0, - "total_count": 0, - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/reactions" - }, - "repository_url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "state": "open", - "state_reason": null, - "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", - "title": "New issue to generate event payloads for static tests", - "updated_at": "2023-01-30T18:20:53Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", - "user": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - }, - "label": { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - }, - "repository": { - "allow_forking": true, - "archive_url": "https://api.github.com/repos/Azure/azure-sdk-fake/{archive_format}{/ref}", - "archived": false, - "assignees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/assignees{/user}", - "blobs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/Azure/azure-sdk-fake/branches{/branch}", - "clone_url": "https://github.com/Azure/azure-sdk-fake.git", - "collaborators_url": "https://api.github.com/repos/Azure/azure-sdk-fake/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/comments{/number}", - "commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/commits{/sha}", - "compare_url": "https://api.github.com/repos/Azure/azure-sdk-fake/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contents/{+path}", - "contributors_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contributors", - "created_at": "2022-06-27T16:19:29Z", - "default_branch": "main", - "deployments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/deployments", - "description": "Tools repository leveraged by the Azure SDK team.", - "disabled": false, - "downloads_url": "https://api.github.com/repos/Azure/azure-sdk-fake/downloads", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/events", - "fork": true, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/forks", - "full_name": "Azure/azure-sdk-fake", - "git_commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/tags{/sha}", - "git_url": "git://github.com/Azure/azure-sdk-fake.git", - "has_discussions": false, - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_projects": true, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/hooks", - "html_url": "https://github.com/Azure/azure-sdk-fake", - "id": 507980610, - "is_template": false, - "issue_comment_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/events{/number}", - "issues_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues{/number}", - "keys_url": "https://api.github.com/repos/Azure/azure-sdk-fake/keys{/key_id}", - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels{/name}", - "language": "C#", - "languages_url": "https://api.github.com/repos/Azure/azure-sdk-fake/languages", - "license": { - "key": "mit", - "name": "MIT License", - "node_id": "MDc6TGljZW5zZTEz", - "spdx_id": "MIT", - "url": "https://api.github.com/licenses/mit" - }, - "merges_url": "https://api.github.com/repos/Azure/azure-sdk-fake/merges", - "milestones_url": "https://api.github.com/repos/Azure/azure-sdk-fake/milestones{/number}", - "mirror_url": null, - "name": "azure-sdk-fake", - "node_id": "R_kgDOHkcrQg", - "notifications_url": "https://api.github.com/repos/Azure/azure-sdk-fake/notifications{?since,all,participating}", - "open_issues": 6, - "open_issues_count": 6, - "owner": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/Azure/azure-sdk-fake/pulls{/number}", - "pushed_at": "2023-01-27T16:33:00Z", - "releases_url": "https://api.github.com/repos/Azure/azure-sdk-fake/releases{/id}", - "size": 29098, - "ssh_url": "git@github.com:Azure/azure-sdk-fake.git", - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/stargazers", - "statuses_url": "https://api.github.com/repos/Azure/azure-sdk-fake/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscribers", - "subscription_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscription", - "svn_url": "https://github.com/Azure/azure-sdk-fake", - "tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/tags", - "teams_url": "https://api.github.com/repos/Azure/azure-sdk-fake/teams", - "topics": [], - "trees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/trees{/sha}", - "updated_at": "2023-01-23T19:54:18Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "visibility": "public", - "watchers": 0, - "watchers_count": 0, - "web_commit_signoff_required": false - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } -} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention_has_service_attention.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention_has_service_attention.json deleted file mode 100644 index 450cb7bb62b..00000000000 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention_has_service_attention.json +++ /dev/null @@ -1,284 +0,0 @@ -{ - "action": "unlabeled", - "issue": { - "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], - "author_association": "OWNER", - "body": null, - "closed_at": null, - "comments": 0, - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/comments", - "created_at": "2023-01-27T17:01:30Z", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/events", - "html_url": "https://github.com/Azure/azure-sdk-fake/issues/14", - "id": 1560095682, - "labels": [ - { - "color": "d73a4a", - "default": true, - "description": "Something isn't working", - "id": 4273699693, - "name": "bug", - "node_id": "LA_kwDOHkcrQs7-u3tt", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/bug" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4976488245, - "name": "customer-reported", - "node_id": "LA_kwDOHkcrQs8AAAABKJ8vNQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/customer-reported" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4704569627, - "name": "needs-triage", - "node_id": "LA_kwDOHkcrQs8AAAABGGoJGw", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/needs-triage" - }, - { - "color": "9C7082", - "default": false, - "description": "fake label for testing", - "id": 5095712487, - "name": "FakeLabel1", - "node_id": "LA_kwDOHkcrQs8AAAABL7pm5w", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/FakeLabel1" - }, - { - "color": "F75BB2", - "default": false, - "description": "", - "id": 5095715984, - "name": "Service Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7p0kA", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/Service%20Attention" - } - ], - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/labels{/name}", - "locked": false, - "milestone": null, - "node_id": "I_kwDOHkcrQs5c_SvC", - "number": 14, - "performed_via_github_app": null, - "reactions": { - "+1": 0, - "-1": 0, - "confused": 0, - "eyes": 0, - "heart": 0, - "hooray": 0, - "laugh": 0, - "rocket": 0, - "total_count": 0, - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/reactions" - }, - "repository_url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "state": "open", - "state_reason": null, - "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", - "title": "New test issue to generate event payloads", - "updated_at": "2023-01-30T16:36:13Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", - "user": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - }, - "label": { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - }, - "repository": { - "allow_forking": true, - "archive_url": "https://api.github.com/repos/Azure/azure-sdk-fake/{archive_format}{/ref}", - "archived": false, - "assignees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/assignees{/user}", - "blobs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/Azure/azure-sdk-fake/branches{/branch}", - "clone_url": "https://github.com/Azure/azure-sdk-fake.git", - "collaborators_url": "https://api.github.com/repos/Azure/azure-sdk-fake/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/comments{/number}", - "commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/commits{/sha}", - "compare_url": "https://api.github.com/repos/Azure/azure-sdk-fake/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contents/{+path}", - "contributors_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contributors", - "created_at": "2022-06-27T16:19:29Z", - "default_branch": "main", - "deployments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/deployments", - "description": "Tools repository leveraged by the Azure SDK team.", - "disabled": false, - "downloads_url": "https://api.github.com/repos/Azure/azure-sdk-fake/downloads", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/events", - "fork": true, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/forks", - "full_name": "Azure/azure-sdk-fake", - "git_commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/tags{/sha}", - "git_url": "git://github.com/Azure/azure-sdk-fake.git", - "has_discussions": false, - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_projects": true, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/hooks", - "html_url": "https://github.com/Azure/azure-sdk-fake", - "id": 507980610, - "is_template": false, - "issue_comment_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/events{/number}", - "issues_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues{/number}", - "keys_url": "https://api.github.com/repos/Azure/azure-sdk-fake/keys{/key_id}", - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels{/name}", - "language": "C#", - "languages_url": "https://api.github.com/repos/Azure/azure-sdk-fake/languages", - "license": { - "key": "mit", - "name": "MIT License", - "node_id": "MDc6TGljZW5zZTEz", - "spdx_id": "MIT", - "url": "https://api.github.com/licenses/mit" - }, - "merges_url": "https://api.github.com/repos/Azure/azure-sdk-fake/merges", - "milestones_url": "https://api.github.com/repos/Azure/azure-sdk-fake/milestones{/number}", - "mirror_url": null, - "name": "azure-sdk-fake", - "node_id": "R_kgDOHkcrQg", - "notifications_url": "https://api.github.com/repos/Azure/azure-sdk-fake/notifications{?since,all,participating}", - "open_issues": 6, - "open_issues_count": 6, - "owner": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/Azure/azure-sdk-fake/pulls{/number}", - "pushed_at": "2023-01-27T16:33:00Z", - "releases_url": "https://api.github.com/repos/Azure/azure-sdk-fake/releases{/id}", - "size": 29098, - "ssh_url": "git@github.com:Azure/azure-sdk-fake.git", - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/stargazers", - "statuses_url": "https://api.github.com/repos/Azure/azure-sdk-fake/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscribers", - "subscription_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscription", - "svn_url": "https://github.com/Azure/azure-sdk-fake", - "tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/tags", - "teams_url": "https://api.github.com/repos/Azure/azure-sdk-fake/teams", - "topics": [], - "trees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/trees{/sha}", - "updated_at": "2023-01-23T19:54:18Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "visibility": "public", - "watchers": 0, - "watchers_count": 0, - "web_commit_signoff_required": false - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } -} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_has_CXP_attention.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_has_CXP_attention.json deleted file mode 100644 index 3e59398e358..00000000000 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_has_CXP_attention.json +++ /dev/null @@ -1,284 +0,0 @@ -{ - "action": "unlabeled", - "issue": { - "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], - "author_association": "OWNER", - "body": null, - "closed_at": null, - "comments": 0, - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/comments", - "created_at": "2023-01-27T17:01:30Z", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/events", - "html_url": "https://github.com/Azure/azure-sdk-fake/issues/14", - "id": 1560095682, - "labels": [ - { - "color": "d73a4a", - "default": true, - "description": "Something isn't working", - "id": 4273699693, - "name": "bug", - "node_id": "LA_kwDOHkcrQs7-u3tt", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/bug" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4976488245, - "name": "customer-reported", - "node_id": "LA_kwDOHkcrQs8AAAABKJ8vNQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/customer-reported" - }, - { - "color": "ededed", - "default": false, - "description": null, - "id": 4704569627, - "name": "needs-triage", - "node_id": "LA_kwDOHkcrQs8AAAABGGoJGw", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/needs-triage" - }, - { - "color": "9C7082", - "default": false, - "description": "fake label for testing", - "id": 5095712487, - "name": "FakeLabel1", - "node_id": "LA_kwDOHkcrQs8AAAABL7pm5w", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/FakeLabel1" - }, - { - "color": "FDEB99", - "default": false, - "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" - } - ], - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/labels{/name}", - "locked": false, - "milestone": null, - "node_id": "I_kwDOHkcrQs5c_SvC", - "number": 14, - "performed_via_github_app": null, - "reactions": { - "+1": 0, - "-1": 0, - "confused": 0, - "eyes": 0, - "heart": 0, - "hooray": 0, - "laugh": 0, - "rocket": 0, - "total_count": 0, - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/reactions" - }, - "repository_url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "state": "open", - "state_reason": null, - "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", - "title": "New test issue to generate event payloads", - "updated_at": "2023-01-30T16:37:38Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", - "user": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - }, - "label": { - "color": "F75BB2", - "default": false, - "description": "", - "id": 5095715984, - "name": "Service Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7p0kA", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/Service%20Attention" - }, - "repository": { - "allow_forking": true, - "archive_url": "https://api.github.com/repos/Azure/azure-sdk-fake/{archive_format}{/ref}", - "archived": false, - "assignees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/assignees{/user}", - "blobs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/Azure/azure-sdk-fake/branches{/branch}", - "clone_url": "https://github.com/Azure/azure-sdk-fake.git", - "collaborators_url": "https://api.github.com/repos/Azure/azure-sdk-fake/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/comments{/number}", - "commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/commits{/sha}", - "compare_url": "https://api.github.com/repos/Azure/azure-sdk-fake/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contents/{+path}", - "contributors_url": "https://api.github.com/repos/Azure/azure-sdk-fake/contributors", - "created_at": "2022-06-27T16:19:29Z", - "default_branch": "main", - "deployments_url": "https://api.github.com/repos/Azure/azure-sdk-fake/deployments", - "description": "Tools repository leveraged by the Azure SDK team.", - "disabled": false, - "downloads_url": "https://api.github.com/repos/Azure/azure-sdk-fake/downloads", - "events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/events", - "fork": true, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/forks", - "full_name": "Azure/azure-sdk-fake", - "git_commits_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/tags{/sha}", - "git_url": "git://github.com/Azure/azure-sdk-fake.git", - "has_discussions": false, - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_projects": true, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/Azure/azure-sdk-fake/hooks", - "html_url": "https://github.com/Azure/azure-sdk-fake", - "id": 507980610, - "is_template": false, - "issue_comment_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/events{/number}", - "issues_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues{/number}", - "keys_url": "https://api.github.com/repos/Azure/azure-sdk-fake/keys{/key_id}", - "labels_url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels{/name}", - "language": "C#", - "languages_url": "https://api.github.com/repos/Azure/azure-sdk-fake/languages", - "license": { - "key": "mit", - "name": "MIT License", - "node_id": "MDc6TGljZW5zZTEz", - "spdx_id": "MIT", - "url": "https://api.github.com/licenses/mit" - }, - "merges_url": "https://api.github.com/repos/Azure/azure-sdk-fake/merges", - "milestones_url": "https://api.github.com/repos/Azure/azure-sdk-fake/milestones{/number}", - "mirror_url": null, - "name": "azure-sdk-fake", - "node_id": "R_kgDOHkcrQg", - "notifications_url": "https://api.github.com/repos/Azure/azure-sdk-fake/notifications{?since,all,participating}", - "open_issues": 6, - "open_issues_count": 6, - "owner": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/Azure/azure-sdk-fake/pulls{/number}", - "pushed_at": "2023-01-27T16:33:00Z", - "releases_url": "https://api.github.com/repos/Azure/azure-sdk-fake/releases{/id}", - "size": 29098, - "ssh_url": "git@github.com:Azure/azure-sdk-fake.git", - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/stargazers", - "statuses_url": "https://api.github.com/repos/Azure/azure-sdk-fake/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscribers", - "subscription_url": "https://api.github.com/repos/Azure/azure-sdk-fake/subscription", - "svn_url": "https://github.com/Azure/azure-sdk-fake", - "tags_url": "https://api.github.com/repos/Azure/azure-sdk-fake/tags", - "teams_url": "https://api.github.com/repos/Azure/azure-sdk-fake/teams", - "topics": [], - "trees_url": "https://api.github.com/repos/Azure/azure-sdk-fake/git/trees{/sha}", - "updated_at": "2023-01-23T19:54:18Z", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake", - "visibility": "public", - "watchers": 0, - "watchers_count": 0, - "web_commit_signoff_required": false - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } -} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_no_assignees.json similarity index 81% rename from tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json rename to tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_no_assignees.json index 9aa49ed265d..af1fa265f90 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_CXP_attention.json +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/Tests.JsonEventPayloads/ManualTriageAfterExternalAssignment_issue_unlabeled_service_attention_no_assignees.json @@ -2,48 +2,8 @@ "action": "unlabeled", "issue": { "active_lock_reason": null, - "assignee": { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - }, - "assignees": [ - { - "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", - "events_url": "https://api.github.com/users/FakeUser1/events{/privacy}", - "followers_url": "https://api.github.com/users/FakeUser1/followers", - "following_url": "https://api.github.com/users/FakeUser1/following{/other_user}", - "gists_url": "https://api.github.com/users/FakeUser1/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/FakeUser1", - "id": 13556087, - "login": "FakeUser1", - "node_id": "MDQ6VXNlcjEzNTU2MDg3", - "organizations_url": "https://api.github.com/users/FakeUser1/orgs", - "received_events_url": "https://api.github.com/users/FakeUser1/received_events", - "repos_url": "https://api.github.com/users/FakeUser1/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/FakeUser1/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/FakeUser1/subscriptions", - "type": "User", - "url": "https://api.github.com/users/FakeUser1" - } - ], + "assignee": null, + "assignees": [], "author_association": "OWNER", "body": null, "closed_at": null, @@ -114,7 +74,7 @@ "state_reason": null, "timeline_url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14/timeline", "title": "New test issue to generate event payloads", - "updated_at": "2023-01-30T16:36:13Z", + "updated_at": "2023-01-30T16:37:38Z", "url": "https://api.github.com/repos/Azure/azure-sdk-fake/issues/14", "user": { "avatar_url": "https://avatars.githubusercontent.com/u/13556087?v=4", @@ -138,13 +98,13 @@ } }, "label": { - "color": "FDEB99", + "color": "F75BB2", "default": false, "description": "", - "id": 5095753141, - "name": "CXP Attention", - "node_id": "LA_kwDOHkcrQs8AAAABL7sFtQ", - "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/CXP%20Attention" + "id": 5095715984, + "name": "Service Attention", + "node_id": "LA_kwDOHkcrQs8AAAABL7p0kA", + "url": "https://api.github.com/repos/Azure/azure-sdk-fake/labels/Service%20Attention" }, "repository": { "allow_forking": true, diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/LabelConstants.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/LabelConstants.cs index 3c96389b383..f7097f415ce 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/LabelConstants.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/LabelConstants.cs @@ -14,7 +14,6 @@ public class LabelConstants { public const string CommunityContribution = "Community Contribution"; public const string CustomerReported = "customer-reported"; - public const string CXPAttention = "CXP Attention"; public const string IssueAddressed = "issue-addressed"; public const string NeedsAuthorFeedback = "needs-author-feedback"; public const string NeedsTeamAttention = "needs-team-attention"; diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/EventProcessing/IssueProcessing.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/EventProcessing/IssueProcessing.cs index f4587c0e31a..06728e24b28 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/EventProcessing/IssueProcessing.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/EventProcessing/IssueProcessing.cs @@ -27,7 +27,6 @@ public static async Task ProcessIssueEvent(GitHubEventClient gitHubEventClient, await InitialIssueTriage(gitHubEventClient, issueEventPayload); ManualIssueTriage(gitHubEventClient, issueEventPayload); ServiceAttention(gitHubEventClient, issueEventPayload); - CXPAttention(gitHubEventClient, issueEventPayload); ManualTriageAfterExternalAssignment(gitHubEventClient, issueEventPayload); RequireAttentionForNonMilestone(gitHubEventClient, issueEventPayload); AuthorFeedbackNeeded(gitHubEventClient, issueEventPayload); @@ -188,46 +187,13 @@ public static void ServiceAttention(GitHubEventClient gitHubEventClient, IssueEv } } - /// - /// CXP Attention - /// Trigger: issue labeled - /// Conditions: Issue is open - /// Label being added is "CXP Attention" - /// Does not have "Service-Attention" label - /// Resulting Action: Add issue comment "Thank you for your feedback. This has been routed to the support team for assistance." - /// Note: The comment added for this rule seems odd, since there's not much of anything of consequence in the comment. - /// The CXP team has a dashboard and automation specifically tied to the CXP Attention label. The comment is necessary to count - /// as an initial response for SLA metrics. - /// - /// Authenticated GitHubEventClient - /// IssueEventGitHubPayload deserialized from the json event payload - public static void CXPAttention(GitHubEventClient gitHubEventClient, IssueEventGitHubPayload issueEventPayload) - { - if (gitHubEventClient.RulesConfiguration.RuleEnabled(RulesConstants.CXPAttention)) - { - if (issueEventPayload.Action == ActionConstants.Labeled) - { - - if (issueEventPayload.Issue.State == ItemState.Open && - issueEventPayload.Label.Name.Equals(LabelConstants.CXPAttention) && - !LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.ServiceAttention)) - { - string issueComment = "Thank you for your feedback. This has been routed to the support team for assistance."; - gitHubEventClient.CreateComment(issueEventPayload.Repository.Id, issueEventPayload.Issue.Number, issueComment); - } - } - } - } - /// /// Manual Triage After External Assignment /// Trigger: issue unlabeled /// Conditions: Issue is open + /// Issue is unassigned /// Has "customer-reported" label - /// Label removed is "Service Attention" OR "CXP Attention" - /// Issue does not have "Service Attention" OR "CXP Attention" - /// (in other words if both labels are on the issue and one is removed, this - /// shouldn't process) + /// Label removed is "Service Attention" /// Resulting Action: Add "needs-team-triage" label /// /// Authenticated GitHubEventClient @@ -239,12 +205,10 @@ public static void ManualTriageAfterExternalAssignment(GitHubEventClient gitHubE if (issueEventPayload.Action == ActionConstants.Unlabeled) { if (issueEventPayload.Issue.State == ItemState.Open && + issueEventPayload.Issue.Assignee == null && + issueEventPayload.Label.Name.Equals(LabelConstants.ServiceAttention) && LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.CustomerReported) && - (issueEventPayload.Label.Name.Equals(LabelConstants.CXPAttention) || - issueEventPayload.Label.Name.Equals(LabelConstants.ServiceAttention)) && - !LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.NeedsTeamTriage) && - !LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.CXPAttention) && - !LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.ServiceAttention)) + !LabelUtils.HasLabel(issueEventPayload.Issue.Labels, LabelConstants.NeedsTeamTriage)) { gitHubEventClient.AddLabel(LabelConstants.NeedsTeamTriage); } @@ -273,7 +237,7 @@ public static void ResetIssueActivity(GitHubEventClient gitHubEventClient, Issue /// Trigger: issue reopened/edited, issue_comment created /// Conditions: Issue is open OR Issue is being reopened /// Issue has "no-recent-activity" label - /// User modifying the issue is NOT a known bot + /// User modifying the issue is NOT a known bot /// Resulting Action: Remove "no-recent-activity" label /// /// Authenticated GitHubEventClient @@ -335,13 +299,13 @@ public static void RequireAttentionForNonMilestone(GitHubEventClient gitHubEvent /// Trigger: issue labeled /// Conditions: Issue is open /// Label added is "needs-author-feedback" - /// Resulting Action: + /// Resulting Action: /// Remove "needs-triage" label /// Remove "needs-team-triage" label /// Remove "needs-team-attention" label /// Create the following comment - /// "Hi @{issueAuthor}. Thank you for opening this issue and giving us the opportunity to assist. To help our - /// team better understand your issue and the details of your scenario please provide a response to the question + /// "Hi @{issueAuthor}. Thank you for opening this issue and giving us the opportunity to assist. To help our + /// team better understand your issue and the details of your scenario please provide a response to the question /// asked above or the information requested above. This will help us more accurately address your issue." /// /// Authenticated GitHubEventClient @@ -381,7 +345,7 @@ public static void AuthorFeedbackNeeded(GitHubEventClient gitHubEventClient, Iss /// Trigger: issue labeled /// Conditions: Issue is open /// Label added is "issue-addressed" - /// Resulting Action: + /// Resulting Action: /// Remove "needs-triage" label if it exists on the issue /// Remove "needs-team-triage" label if it exists on the issue /// Remove "needs-team-attention" label if it exists on the issue @@ -440,7 +404,7 @@ public static void IssueAddressed(GitHubEventClient gitHubEventClient, IssueEven /// "CXP Attention" /// "needs-triage" /// "needs-team-triage" - /// Resulting Action: + /// Resulting Action: /// Remove "issue-addressed" label /// /// Authenticated GitHubEventClient @@ -457,7 +421,6 @@ public static void IssueAddressedReset(GitHubEventClient gitHubEventClient, Issu if (issueEventPayload.Label.Name == LabelConstants.NeedsTeamAttention || issueEventPayload.Label.Name == LabelConstants.NeedsAuthorFeedback || issueEventPayload.Label.Name == LabelConstants.ServiceAttention || - issueEventPayload.Label.Name == LabelConstants.CXPAttention || issueEventPayload.Label.Name == LabelConstants.NeedsTriage || issueEventPayload.Label.Name == LabelConstants.NeedsTeamTriage) { diff --git a/tools/github-event-processor/RULES.md b/tools/github-event-processor/RULES.md index 776a3d74d9c..96d0fe7dd35 100644 --- a/tools/github-event-processor/RULES.md +++ b/tools/github-event-processor/RULES.md @@ -25,7 +25,7 @@ /sdk/eventgrid/ @user1 @user2 @user3 @user4 ``` -- **Azure SDK team owners**: Each package in our repository that is owned by our team will need to have an association tracking a service label to one or more GitHub handles. This is an unofficial mapping that does not necessarily correlate to authoritative "Azure SDK Team" membership, such as that represented by our security groups or the "[azure-sdk-team](https://repos.opensource.microsoft.com/orgs/Azure/teams/azure-sdk-team)" GitHub team. That said, using one of these groups to validate membership on top of the label association would be just fine. *Right now, this does not exist, this would require changes to CODEOWNERS which are still pending.* If it exists, the Service Label to user mapping is in the repository's CODEOWNERS file. +- **Azure SDK team owners**: Each package in our repository that is owned by our team will need to have an association tracking a service label and category label pair to one or more GitHub handles. This is an unofficial mapping that does not necessarily correlate to authoritative "Azure SDK Team" membership, such as that represented by our security groups or the "[azure-sdk-team](https://repos.opensource.microsoft.com/orgs/Azure/teams/azure-sdk-team)" GitHub team. That said, using one of these groups to validate membership on top of the label association would be just fine. *Right now, this does not exist, this would require changes to CODEOWNERS which are still pending.* If it exists, the Service Label to user mapping is in the repository's CODEOWNERS file. Example: @@ -137,7 +137,7 @@ This is a stand-alone service providing a REST API which requires a service key - Assign returned labels to the issue - Add "needs-team-attention" label to the issue - IF service label is associated with an Azure SDK team member: + IF service and category labels are associated with an Azure SDK team member: IF a single team member: - Assign team member to the issue ELSE @@ -148,7 +148,7 @@ This is a stand-alone service providing a REST API which requires a service key - Create the following comment "Thank you for your feedback. Tagging and routing to the team member best able to assist." ELSE - - Add "CXP Attention" label to the issue + - Add "Service Attention" label to the issue - Create the following comment - "Thank you for your feedback. This has been routed to the support team for assistance." ELSE @@ -190,24 +190,6 @@ This is a stand-alone service providing a REST API which requires a service key - Create the following comment, mentioning the service team contacts - "Thanks for the feedback! We are routing this to the appropriate team for follow-up. cc ${mentionees}." -## CXP Attention - -### Trigger - -- Issue modified for: - - Label added - -### Criteria - -- Issue is open -- Label added is "CXP Attention" -- Issues does NOT have label "Service Attention" - -### Actions - -- Create the following comment - - "Thank you for your feedback. This has been routed to the support team for assistance." - ## Manual triage after external assignment ### Trigger @@ -218,9 +200,9 @@ This is a stand-alone service providing a REST API which requires a service key ### Criteria - Issue is open +- Issue is not assigned - Issue has "customer-reported" label -- Label removed is "Service Attention" OR "CXP Attention" -- Issue does not have "Service Attention" OR "CXP Attention" +- Label removed is "Service Attention" ### Actions @@ -417,7 +399,6 @@ OR - "needs-team-attention" - "needs-author-feedback" - "Service Attention" - - "CXP Attention" - "needs-triage" - "needs-team-triage" diff --git a/tools/github-event-processor/YmlAndConfigFiles/event-processor.config b/tools/github-event-processor/YmlAndConfigFiles/event-processor.config index 3bfc0b9d432..aeb6e4c1b79 100644 --- a/tools/github-event-processor/YmlAndConfigFiles/event-processor.config +++ b/tools/github-event-processor/YmlAndConfigFiles/event-processor.config @@ -2,7 +2,6 @@ "InitialIssueTriage": "On", "ManualIssueTriage": "On", "ServiceAttention": "On", - "CXPAttention": "On", "ManualTriageAfterExternalAssignment": "On", "RequireAttentionForNonMilestone": "On", "AuthorFeedbackNeeded": "On", diff --git a/tools/github/data/common-labels.csv b/tools/github/data/common-labels.csv index fe0b520d63b..7b2a309ae76 100644 --- a/tools/github/data/common-labels.csv +++ b/tools/github/data/common-labels.csv @@ -93,7 +93,6 @@ Cost Management - RIandShowBack,"All issues in cost management associated with R Custom Providers,,e99695 Customer Insights,,e99695 CycleCloud,,e99695 -CXP Attention,The Azure CXP Support Team is responsible for this issue.,10066b Data Bricks,,e99695 Data Catalog,,e99695 Data Factory,,e99695 From 2e0c42710d06d110f5970dcffedbf63295fd854f Mon Sep 17 00:00:00 2001 From: Jesse Squire Date: Fri, 29 Sep 2023 13:30:22 -0700 Subject: [PATCH 45/93] [JimBot] Remove CXP Attention (#7034) Removing config for the dead CXP Attention rule. --- .github/event-processor.config | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/event-processor.config b/.github/event-processor.config index a2f516f092f..aa8d4f71b05 100644 --- a/.github/event-processor.config +++ b/.github/event-processor.config @@ -2,7 +2,6 @@ "InitialIssueTriage": "On", "ManualIssueTriage": "On", "ServiceAttention": "Off", - "CXPAttention": "Off", "ManualTriageAfterExternalAssignment": "Off", "RequireAttentionForNonMilestone": "Off", "AuthorFeedbackNeeded": "Off", From 95ba08ba5ab41ce9ede6a12a9e2f10bd51d44d84 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 2 Oct 2023 14:59:40 -0700 Subject: [PATCH 46/93] Remove get-codeowners.ps1 from docs automation (#7040) * Remove codeowners resolution logic from docs scripts * Remove parameters that aren't needed * Fix test expectations to exclude metadata that's been removed * Remove auth information from update-docsms-metadata.yml * Fix notification-configuration.sln --- .../actual/latest/service-name-4-index.md | 15 +- .../inputs/actual/latest/service-name-4.md | 22 +- .../actual/preview/service-name-5-index.md | 14 +- .../inputs/actual/preview/service-name-5.md | 8 + .../actual/preview/service-name-6-index.md | 15 +- .../inputs/actual/preview/service-name-6.md | 22 +- .../inputs/expected/latest/service-name-2.md | 4 +- .../inputs/expected/latest/service-name-4.md | 4 +- .../inputs/expected/preview/service-name-1.md | 4 +- .../inputs/expected/preview/service-name-3.md | 4 +- .../inputs/expected/preview/service-name-5.md | 4 +- .../inputs/expected/preview/service-name-6.md | 4 +- .../get-codeowners/get-codeowners.tests.ps1 | 46 -- .../steps/update-docsms-metadata.yml | 5 +- .../scripts/Helpers/Metadata-Helpers.ps1 | 54 +- ...ervice-Level-Readme-Automation-Helpers.ps1 | 73 +- .../Service-Level-Readme-Automation.ps1 | 45 +- eng/common/scripts/Update-DocsMsMetadata.ps1 | 37 +- eng/common/scripts/get-codeowners.lib.ps1 | 138 ---- eng/common/scripts/get-codeowners.ps1 | 18 - ....Sdk.Tools.RetrieveCodeOwners.Tests.csproj | 27 - .../CodeownersManualAnalysisTests.cs | 623 ------------------ .../ConsoleOutput.cs | 70 -- .../RetrieveCodeOwnersProgramTests.cs | 214 ------ .../TestData/InputDir/a.txt | 0 .../TestData/InputDir/b.txt | 12 - .../TestData/InputDir/baz/cor/c.txt | 0 .../TestData/InputDir/baz_.txt | 0 .../TestData/InputDir/cor.txt | 0 .../TestData/InputDir/cor/gra/a.txt | 0 .../TestData/InputDir/cor2/a.txt | 0 .../TestData/InputDir/foo/a.txt | 12 - .../TestData/InputDir/foo/b.txt | 12 - .../TestData/InputDir/foo/bar/a.txt | 0 .../TestData/InputDir/foo/bar/b.txt | 0 .../TestData/InputDir/qux/abc/d.txt | 0 .../TestData/test_CODEOWNERS | 16 - .../Azure.Sdk.Tools.RetrieveCodeOwners.csproj | 21 - .../Program.cs | 156 ----- tools/code-owners-parser/CodeOwnersParser.sln | 82 +-- tools/code-owners-parser/ci.yml | 33 - .../notification-configuration.sln | 122 ++-- 42 files changed, 216 insertions(+), 1720 deletions(-) delete mode 100644 eng/common-tests/get-codeowners/get-codeowners.tests.ps1 delete mode 100644 eng/common/scripts/get-codeowners.lib.ps1 delete mode 100644 eng/common/scripts/get-codeowners.ps1 delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/CodeownersManualAnalysisTests.cs delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/RetrieveCodeOwnersProgramTests.cs delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/b.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz/cor/c.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz_.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor/gra/a.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor2/a.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/a.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/b.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/a.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/b.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/qux/abc/d.txt delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Azure.Sdk.Tools.RetrieveCodeOwners.csproj delete mode 100644 tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs delete mode 100644 tools/code-owners-parser/ci.yml diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4-index.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4-index.md index fc167d680f7..42edb02de07 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4-index.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4-index.md @@ -1,8 +1,7 @@ -| Reference | Package | Source | -|---|---|---| -|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[Github](github.com)| -|Communication Identity|[azure-communication-identity](azure-communication-identity/test)|[Github](github.com/blob/main/sdk/communication/azure-communication-identity)| -|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[Github](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| -|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[Github](github.com)| -|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[Github](github.com)| -|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[Github](github.com)| +| Reference | Package | Source | +|---|---|---| +|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[GitHub](github.com)| +|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[GitHub](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| +|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[GitHub](github.com)| +|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[GitHub](github.com)| +|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[GitHub](github.com)| diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4.md index 71e0081a2a3..9a350e48628 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/latest/service-name-4.md @@ -1,13 +1,11 @@ ---- -title: Azure service name 4 SDK for Unknown -description: Reference for Azure service name 4 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 -ms.topic: reference -ms.devlang: Unknown -ms.service: ms-service ---- -# Azure service name 4 SDK for Unknown - preview -## Packages - preview +--- +title: Azure service name 4 SDK for Unknown +description: Reference for Azure service name 4 SDK for Unknown +ms.date: 2022-11-01 +ms.topic: reference +ms.devlang: Unknown +ms.service: ms-service +--- +# Azure service name 4 SDK for Unknown - preview +## Packages - preview [!INCLUDE [packages](service-name-3-index.md)] \ No newline at end of file diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5-index.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5-index.md index f52806c9ca8..42edb02de07 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5-index.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5-index.md @@ -1,7 +1,7 @@ -| Reference | Package | Source | -|---|---|---| -|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[Github](github.com)| -|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[Github](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| -|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[Github](github.com)| -|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[Github](github.com)| -|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[Github](github.com)| +| Reference | Package | Source | +|---|---|---| +|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[GitHub](github.com)| +|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[GitHub](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| +|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[GitHub](github.com)| +|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[GitHub](github.com)| +|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[GitHub](github.com)| diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5.md index c27e18076ac..ac9e6e5c387 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-5.md @@ -1 +1,9 @@ +--- +title: Azure service name 5 SDK for Unknown +description: Reference for Azure service name 5 SDK for Unknown +ms.date: 2022-11-01 +ms.topic: reference +ms.devlang: Unknown +ms.service: ms-service +--- This is testing \ No newline at end of file diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6-index.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6-index.md index fc167d680f7..42edb02de07 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6-index.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6-index.md @@ -1,8 +1,7 @@ -| Reference | Package | Source | -|---|---|---| -|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[Github](github.com)| -|Communication Identity|[azure-communication-identity](azure-communication-identity/test)|[Github](github.com/blob/main/sdk/communication/azure-communication-identity)| -|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[Github](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| -|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[Github](github.com)| -|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[Github](github.com)| -|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[Github](github.com)| +| Reference | Package | Source | +|---|---|---| +|Communication Chat|[azure-communication-chat](azure-communication-chat/test)|[GitHub](github.com)| +|Communication Network Traversal|[azure-communication-networktraversal](azure-communication-networktraversal/test)|[GitHub](github.com/blob/main/sdk/communication/azure-communication-networktraversal)| +|Communication Phone Numbers|[azure-communication-phonenumbers](azure-communication-phonenumbers/test)|[GitHub](github.com)| +|Communication Sms|[azure-communication-sms](azure-communication-sms/test)|[GitHub](github.com)| +|Resource Management - Communication|[azure-mgmt-communication](azure-mgmt-communication/test)|[GitHub](github.com)| diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6.md index 2286efd9885..97e45a087fe 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/actual/preview/service-name-6.md @@ -1,13 +1,11 @@ ---- -title: Azure service name 6 SDK for Unknown -description: Reference for Azure service name 6 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 -ms.topic: reference -ms.devlang: Unknown -ms.service: ms-service ---- -# Azure service name 6 SDK for Unknown - preview -## Packages - preview +--- +title: Azure service name 6 SDK for Unknown +description: Reference for Azure service name 6 SDK for Unknown +ms.date: 2022-11-01 +ms.topic: reference +ms.devlang: Unknown +ms.service: ms-service +--- +# Azure service name 6 SDK for Unknown - preview +## Packages - preview [!INCLUDE [packages](service-name-5-index.md)] \ No newline at end of file diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-2.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-2.md index 730e7192707..7a80b581500 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-2.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-2.md @@ -1,9 +1,7 @@ --- title: Azure service name 2 SDK for Unknown description: Reference for Azure service name 2 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-4.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-4.md index 71e0081a2a3..42b46ff5e39 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-4.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/latest/service-name-4.md @@ -1,9 +1,7 @@ --- title: Azure service name 4 SDK for Unknown description: Reference for Azure service name 4 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-1.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-1.md index 3b4e04c4573..f76eccf77c7 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-1.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-1.md @@ -1,9 +1,7 @@ --- title: Azure service name 1 SDK for Unknown description: Reference for Azure service name 1 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-3.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-3.md index c3c754ba992..81517cab57b 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-3.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-3.md @@ -1,9 +1,7 @@ --- title: Azure service name 3 SDK for Unknown description: Reference for Azure service name 3 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-5.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-5.md index d025b18b8a7..35de40f6d76 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-5.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-5.md @@ -1,9 +1,7 @@ --- title: Azure service name 5 SDK for Unknown description: Reference for Azure service name 5 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-6.md b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-6.md index 2286efd9885..536e64a68e8 100644 --- a/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-6.md +++ b/eng/common-tests/doc-automation/service-readme-generation/inputs/expected/preview/service-name-6.md @@ -1,9 +1,7 @@ --- title: Azure service name 6 SDK for Unknown description: Reference for Azure service name 6 SDK for Unknown -author: github-alias -ms.author: msalias -ms.data: 2022-11-01 +ms.date: 2022-11-01 ms.topic: reference ms.devlang: Unknown ms.service: ms-service diff --git a/eng/common-tests/get-codeowners/get-codeowners.tests.ps1 b/eng/common-tests/get-codeowners/get-codeowners.tests.ps1 deleted file mode 100644 index 8ec82832fee..00000000000 --- a/eng/common-tests/get-codeowners/get-codeowners.tests.ps1 +++ /dev/null @@ -1,46 +0,0 @@ -Import-Module Pester - -BeforeAll { - . $PSScriptRoot/../../common/scripts/get-codeowners.lib.ps1 - - function TestGetCodeowners( - [string] $TargetPath, - [string] $CodeownersFileLocation, - [string[]] $ExpectedOwners - ) - { - Write-Host "Test: Owners for path '$TargetPath' in CODEOWNERS file at path '$CodeownersFileLocation' should be '$ExpectedOwners'" - - $actualOwners = Get-Codeowners ` - -TargetPath $TargetPath ` - -CodeownersFileLocation $CodeownersFileLocation ` - - $actualOwners.Count | Should -Be $ExpectedOwners.Count - for ($i = 0; $i -lt $ExpectedOwners.Count; $i++) { - $ExpectedOwners[$i] | Should -Be $actualOwners[$i] - } - } -} - -Describe "Get Codeowners" -Tag "UnitTest" { - It "Should get Codeowners" -TestCases @( - @{ - # The $PSScriptRoot is assumed to be azure-sdk-tools/eng/common-tests/get-codeowners/get-codeowners.tests.ps1 - codeownersPath = "$PSScriptRoot/../../../.github/CODEOWNERS"; - targetPath = "eng/common/scripts/get-codeowners/get-codeowners.ps1"; - expectedOwners = @("konrad-jamrozik", "weshaggard", "benbp") - }, - @{ - # The $PSScriptRoot is assumed to be azure-sdk-tools/eng/common-tests/get-codeowners/get-codeowners.tests.ps1 - CodeownersPath = "$PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS"; - targetPath = "tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt"; - expectedOwners = @("2star") - } - ) { - $resolvedCodeownersPath = (Resolve-Path $codeownersPath) - TestGetCodeowners ` - -TargetPath $targetPath ` - -CodeownersFileLocation $resolvedCodeownersPath ` - -ExpectedOwners $expectedOwners - } -} \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml index 2635ad47943..63856f160ce 100644 --- a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml +++ b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml @@ -100,10 +100,7 @@ steps: -Language '${{parameters.Language}}' ` -RepoId '${{ parameters.RepoId }}' ` -DocValidationImageId '${{ parameters.DocValidationImageId }}' ` - -PackageSourceOverride '${{ parameters.PackageSourceOverride }}' ` - -TenantId '$(opensource-aad-tenant-id)' ` - -ClientId '$(opensource-aad-app-id)' ` - -ClientSecret '$(opensource-aad-secret)' + -PackageSourceOverride '${{ parameters.PackageSourceOverride }}' displayName: Apply Documentation Updates - template: /eng/common/pipelines/templates/steps/git-push-changes.yml diff --git a/eng/common/scripts/Helpers/Metadata-Helpers.ps1 b/eng/common/scripts/Helpers/Metadata-Helpers.ps1 index bbc9eaa70c0..3df2c0684a7 100644 --- a/eng/common/scripts/Helpers/Metadata-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Metadata-Helpers.ps1 @@ -17,36 +17,6 @@ function Generate-AadToken ($TenantId, $ClientId, $ClientSecret) return $resp.access_token } -function GetMsAliasFromGithub ([string]$TenantId, [string]$ClientId, [string]$ClientSecret, [string]$GithubUser) -{ - # API documentation (out of date): https://github.com/microsoft/opensource-management-portal/blob/main/docs/api.md - $OpensourceAPIBaseURI = "https://repos.opensource.microsoft.com/api/people/links/github/$GithubUser" - - $Headers = @{ - "Content-Type" = "application/json" - "api-version" = "2019-10-01" - } - - try { - $opsAuthToken = Generate-AadToken -TenantId $TenantId -ClientId $ClientId -ClientSecret $ClientSecret - $Headers["Authorization"] = "Bearer $opsAuthToken" - Write-Host "Fetching aad identity for github user: $GithubUser" - $resp = Invoke-RestMethod $OpensourceAPIBaseURI -Method 'GET' -Headers $Headers -MaximumRetryCount 3 - } catch { - Write-Warning $_ - return $null - } - - $resp | Write-Verbose - - if ($resp.aad) { - Write-Host "Fetched aad identity $($resp.aad.alias) for github user $GithubUser. " - return $resp.aad.alias - } - Write-Warning "Failed to retrieve the aad identity from given github user: $GithubName" - return $null -} - function GetAllGithubUsers ([string]$TenantId, [string]$ClientId, [string]$ClientSecret) { # API documentation (out of date): https://github.com/microsoft/opensource-management-portal/blob/main/docs/api.md @@ -70,17 +40,6 @@ function GetAllGithubUsers ([string]$TenantId, [string]$ClientId, [string]$Clien return $resp } -function GetPrimaryCodeOwner ([string]$TargetDirectory) -{ - $codeOwnerArray = &"$PSScriptRoot/../get-codeowners.ps1" -TargetDirectory $TargetDirectory - if ($codeOwnerArray) { - Write-Host "Code Owners are $codeOwnerArray." - return $codeOwnerArray[0] - } - Write-Warning "No code owner found in $TargetDirectory." - return $null -} - function GetDocsMsService($packageInfo, $serviceName) { $service = $serviceName.ToLower().Replace(' ', '').Replace('/', '-') @@ -109,8 +68,13 @@ function compare-and-merge-metadata ($original, $updated) { return $updateMetdata } -function GenerateDocsMsMetadata($originalMetadata, $language, $languageDisplayName, $serviceName, $author, $msAuthor, $msService) -{ +function GenerateDocsMsMetadata( + $originalMetadata, + $language, + $languageDisplayName, + $serviceName, + $msService +) { $langTitle = "Azure $serviceName SDK for $languageDisplayName" $langDescription = "Reference for Azure $serviceName SDK for $languageDisplayName" $date = Get-Date -Format "MM/dd/yyyy" @@ -118,9 +82,7 @@ function GenerateDocsMsMetadata($originalMetadata, $language, $languageDisplayNa $metadataTable = [ordered]@{ "title"= $langTitle "description"= $langDescription - "author"= $author - "ms.author"= $msauthor - "ms.data"= $date + "ms.date"= $date "ms.topic"= "reference" "ms.devlang"= $language "ms.service"= $msService diff --git a/eng/common/scripts/Helpers/Service-Level-Readme-Automation-Helpers.ps1 b/eng/common/scripts/Helpers/Service-Level-Readme-Automation-Helpers.ps1 index 75742426e91..4382b6159f1 100644 --- a/eng/common/scripts/Helpers/Service-Level-Readme-Automation-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Service-Level-Readme-Automation-Helpers.ps1 @@ -1,8 +1,15 @@ -function create-service-readme($readmeFolder, $readmeName, $moniker, $msService, $indexTableLink, $serviceName, $author, $msAuthor) -{ +function create-service-readme( + $readmeFolder, + $readmeName, + $moniker, + $msService, + $indexTableLink, + $serviceName +) { + $readmePath = Join-Path $readmeFolder -ChildPath $readmeName - $content = "" + $content = "" if (Test-Path (Join-Path $readmeFolder -ChildPath $indexTableLink)) { $content = "## Packages - $moniker`r`n" $content += "[!INCLUDE [packages]($indexTableLink)]" @@ -13,8 +20,12 @@ function create-service-readme($readmeFolder, $readmeName, $moniker, $msService, } # Generate the front-matter for docs needs # $Language, $LanguageDisplayName are the variables globally defined in Language-Settings.ps1 - $metadataString = GenerateDocsMsMetadata -language $Language -languageDisplayName $LanguageDisplayName -serviceName $serviceName ` - -author $author -msAuthor $msAuthor -msService $msService + $metadataString = GenerateDocsMsMetadata ` + -language $Language ` + -languageDisplayName $LanguageDisplayName ` + -serviceName $serviceName ` + -msService $msService + Add-Content -Path $readmePath -Value $metadataString -NoNewline # Add tables, conbined client and mgmt together. @@ -34,14 +45,20 @@ function update-metadata-table($readmeFolder, $readmeName, $serviceName, $msServ $restContent = $Matches["content"].trim() $metadata = $Matches["metadata"].trim() } + # $Language, $LanguageDisplayName are the variables globally defined in Language-Settings.ps1 - $metadataString = GenerateDocsMsMetadata -originalMetadata $metadata -language $Language -languageDisplayName $LanguageDisplayName -serviceName $serviceName ` - -author $author -msAuthor $msAuthor -msService $msService + $metadataString = GenerateDocsMsMetadata ` + -originalMetadata $metadata ` + -language $Language ` + -languageDisplayName $LanguageDisplayName ` + -serviceName $serviceName ` + -msService $msService + Set-Content -Path $readmePath -Value "$metadataString$restContent" -NoNewline } function generate-markdown-table($readmeFolder, $readmeName, $packageInfos, $moniker) { - $tableHeader = "| Reference | Package | Source |`r`n|---|---|---|`r`n" + $tableHeader = "| Reference | Package | Source |`r`n|---|---|---|`r`n" $tableContent = "" $packageInfos = $packageInfos | Sort-Object -Property Type,Package # Here is the table, the versioned value will @@ -54,7 +71,7 @@ function generate-markdown-table($readmeFolder, $readmeName, $packageInfos, $mon if (Test-Path "Function:$GetPackageLevelReadmeFn") { $packageLevelReadme = &$GetPackageLevelReadmeFn -packageMetadata $pkg } - + $referenceLink = "[$($pkg.DisplayName)]($packageLevelReadme-readme.md)" if (!(Test-Path (Join-Path $readmeFolder -ChildPath "$packageLevelReadme-readme.md"))) { $referenceLink = $pkg.DisplayName @@ -72,19 +89,41 @@ function generate-markdown-table($readmeFolder, $readmeName, $packageInfos, $mon } } -function generate-service-level-readme($docRepoLocation, $readmeBaseName, $pathPrefix, $packageInfos, $serviceName, $moniker, $author, $msAuthor, $msService) { +function generate-service-level-readme( + $docRepoLocation, + $readmeBaseName, + $pathPrefix, + $packageInfos, + $serviceName, + $moniker, + $msService +) { $readmeFolder = "$docRepoLocation/$pathPrefix/$moniker/" $serviceReadme = "$readmeBaseName.md" $indexReadme = "$readmeBaseName-index.md" + if ($packageInfos) { - generate-markdown-table -readmeFolder $readmeFolder -readmeName $indexReadme -packageInfos $packageInfos -moniker $moniker + generate-markdown-table ` + -readmeFolder $readmeFolder ` + -readmeName $indexReadme ` + -packageInfos $packageInfos ` + -moniker $moniker } + if (!(Test-Path "$readmeFolder$serviceReadme") -and $packageInfos) { - create-service-readme -readmeFolder $readmeFolder -readmeName $serviceReadme -moniker $moniker -msService $msService ` - -indexTableLink $indexReadme -serviceName $serviceName -author $author -msAuthor $msAuthor - } - elseif (Test-Path "$readmeFolder$serviceReadme") { - update-metadata-table -readmeFolder $readmeFolder -readmeName $serviceReadme -serviceName $serviceName ` - -msService $msService -author $author -msAuthor $msAuthor + create-service-readme ` + -readmeFolder $readmeFolder ` + -readmeName $serviceReadme ` + -moniker $moniker ` + -msService $msService ` + -indexTableLink $indexReadme ` + -serviceName $serviceName + + } elseif (Test-Path "$readmeFolder$serviceReadme") { + update-metadata-table ` + -readmeFolder $readmeFolder ` + -readmeName $serviceReadme ` + -serviceName $serviceName ` + -msService $msService } } \ No newline at end of file diff --git a/eng/common/scripts/Service-Level-Readme-Automation.ps1 b/eng/common/scripts/Service-Level-Readme-Automation.ps1 index a03e78e4e22..10dbee36edf 100644 --- a/eng/common/scripts/Service-Level-Readme-Automation.ps1 +++ b/eng/common/scripts/Service-Level-Readme-Automation.ps1 @@ -13,15 +13,6 @@ Generate missing service level readme and updating metadata of the existing ones Location of the documentation repo. This repo may be sparsely checked out depending on the requirements for the domain -.PARAMETER TenantId -The aad tenant id/object id for ms.author. - -.PARAMETER ClientId -The add client id/application id for ms.author. - -.PARAMETER ClientSecret -The client secret of add app for ms.author. - .PARAMETER ReadmeFolderRoot The readme folder root path, use default value here for backward compability. E.g. docs-ref-services in Java, JS, Python, api/overview/azure #> @@ -30,15 +21,6 @@ param( [Parameter(Mandatory = $true)] [string] $DocRepoLocation, - [Parameter(Mandatory = $false)] - [string]$TenantId, - - [Parameter(Mandatory = $false)] - [string]$ClientId, - - [Parameter(Mandatory = $false)] - [string]$ClientSecret, - [Parameter(Mandatory = $false)] [string]$ReadmeFolderRoot = "docs-ref-services", @@ -132,26 +114,17 @@ foreach($moniker in $Monikers) { Write-Host "Building service: $service" $servicePackages = $packagesForService.Values.Where({ $_.ServiceName -eq $service }) $serviceReadmeBaseName = ServiceLevelReadmeNameStyle -serviceName $service - # Github url for source code: e.g. https://github.com/Azure/azure-sdk-for-js - $serviceBaseName = ServiceLevelReadmeNameStyle $service - $author = GetPrimaryCodeOwner -TargetDirectory "/sdk/$serviceBaseName/" - $msauthor = "" - if (!$author) { - LogError "Cannot fetch the author from CODEOWNER file." - $author = "" - } - elseif ($TenantId -and $ClientId -and $ClientSecret) { - $msauthor = GetMsAliasFromGithub -TenantId $tenantId -ClientId $clientId -ClientSecret $clientSecret -GithubUser $author - } - # Default value - if (!$msauthor) { - LogError "No ms.author found for $author. " - $msauthor = $author - } + # Add ability to override # Fetch the service readme name $msService = GetDocsMsService -packageInfo $servicePackages[0] -serviceName $service - generate-service-level-readme -docRepoLocation $DocRepoLocation -readmeBaseName $serviceReadmeBaseName -pathPrefix $ReadmeFolderRoot ` - -packageInfos $servicePackages -serviceName $service -moniker $moniker -author $author -msAuthor $msauthor -msService $msService + generate-service-level-readme ` + -docRepoLocation $DocRepoLocation ` + -readmeBaseName $serviceReadmeBaseName ` + -pathPrefix $ReadmeFolderRoot ` + -packageInfos $servicePackages ` + -serviceName $service ` + -moniker $moniker ` + -msService $msService } } diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 9b665dbc98d..817407f4bc1 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -32,14 +32,6 @@ GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' The docker image id in format of '$containerRegistry/$imageName:$tag' e.g. azuresdkimages.azurecr.io/jsrefautocr:latest -.PARAMETER TenantId -The aad tenant id/object id. - -.PARAMETER ClientId -The add client id/application id. - -.PARAMETER ClientSecret -The client secret of add app. #> param( @@ -59,16 +51,7 @@ param( [string]$DocValidationImageId, [Parameter(Mandatory = $false)] - [string]$PackageSourceOverride, - - [Parameter(Mandatory = $false)] - [string]$TenantId, - - [Parameter(Mandatory = $false)] - [string]$ClientId, - - [Parameter(Mandatory = $false)] - [string]$ClientSecret + [string]$PackageSourceOverride ) Set-StrictMode -Version 3 . (Join-Path $PSScriptRoot common.ps1) @@ -105,28 +88,10 @@ function GetAdjustedReadmeContent($ReadmeContent, $PackageInfo, $PackageMetadata $ReadmeContent = $ReadmeContent -replace $releaseReplaceRegex, $replacementPattern } - # Get the first code owners of the package. - Write-Host "Retrieve the code owner from $($PackageInfo.DirectoryPath)." - $author = GetPrimaryCodeOwner -TargetDirectory $PackageInfo.DirectoryPath - if (!$author) { - $author = "ramya-rao-a" - $msauthor = "ramyar" - } - else { - $msauthor = GetMsAliasFromGithub -TenantId $TenantId -ClientId $ClientId -ClientSecret $ClientSecret -GithubUser $author - } - # Default value - if (!$msauthor) { - $msauthor = $author - } - Write-Host "The author of package: $author" - Write-Host "The ms author of package: $msauthor" $header = @" --- title: $foundTitle keywords: Azure, $Language, SDK, API, $($PackageInfo.Name), $service -author: $author -ms.author: $msauthor ms.date: $date ms.topic: reference ms.devlang: $Language diff --git a/eng/common/scripts/get-codeowners.lib.ps1 b/eng/common/scripts/get-codeowners.lib.ps1 deleted file mode 100644 index 2fc31e3be09..00000000000 --- a/eng/common/scripts/get-codeowners.lib.ps1 +++ /dev/null @@ -1,138 +0,0 @@ -function Get-CodeownersTool([string] $ToolPath, [string] $DevOpsFeed, [string] $ToolVersion) -{ - $codeownersToolCommand = Join-Path $ToolPath "retrieve-codeowners" - Write-Host "Checking for retrieve-codeowners in $ToolPath ..." - # Check if the retrieve-codeowners tool exists or not. - if (Get-Command $codeownersToolCommand -errorAction SilentlyContinue) { - return $codeownersToolCommand - } - if (!(Test-Path $ToolPath)) { - New-Item -ItemType Directory -Path $ToolPath | Out-Null - } - Write-Host "Installing the retrieve-codeowners tool under tool path: $ToolPath ..." - - # Run command under tool path to avoid dotnet tool install command checking .csproj files. - # This is a bug for dotnet tool command. Issue: https://github.com/dotnet/sdk/issues/9623 - Push-Location $ToolPath - Write-Host "Executing: dotnet tool install --tool-path $ToolPath --add-source $DevOpsFeed --version $ToolVersion" - dotnet tool install --tool-path $ToolPath --add-source $DevOpsFeed --version $ToolVersion "Azure.Sdk.Tools.RetrieveCodeOwners" | Out-Null - Pop-Location - # Test to see if the tool properly installed. - if (!(Get-Command $codeownersToolCommand -errorAction SilentlyContinue)) { - Write-Error "The retrieve-codeowners tool is not properly installed. Please check your tool path: $ToolPath" - return - } - return $codeownersToolCommand -} - -<# -.SYNOPSIS -A function that given as input $TargetPath param, returns the owners -of that path, as determined by CODEOWNERS file passed in $CodeownersFileLocation -param. - -.PARAMETER TargetPath -Required*. Path to file or directory whose owners are to be determined from a -CODEOWNERS file. e.g. sdk/core/azure-amqp/ or sdk/core/foo.txt. - -*for backward compatibility, you might provide $TargetDirectory instead. - -.PARAMETER TargetDirectory -Obsolete. Replaced by $TargetPath. Kept for backward-compatibility. -If both $TargetPath and $TargetDirectory are provided, $TargetDirectory is -ignored. - -.PARAMETER CodeownersFileLocation -Optional. An absolute path to the CODEOWNERS file against which the $TargetPath param -will be checked to determine its owners. - -.PARAMETER ToolVersion -Optional. The NuGet package version of the package containing the "retrieve-codeowners" -tool, around which this script is a wrapper. - -.PARAMETER ToolPath -Optional. The place to check the "retrieve-codeowners" tool existence. - -.PARAMETER DevOpsFeed -Optional. The NuGet package feed from which the "retrieve-codeowners" tool is to be installed. - -NuGet feed: -https://dev.azure.com/azure-sdk/public/_artifacts/feed/azure-sdk-for-net/NuGet/Azure.Sdk.Tools.RetrieveCodeOwners - -Pipeline publishing the NuGet package to the feed, "tools - code-owners-parser": -https://dev.azure.com/azure-sdk/internal/_build?definitionId=3188 - -.PARAMETER VsoVariable -Optional. If provided, the determined owners, based on $TargetPath matched against CODEOWNERS file at $CodeownersFileLocation, -will be output to Azure DevOps pipeline log as variable named $VsoVariable. - -Reference: -https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch -https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#logging-command-format - -.PARAMETER IncludeNonUserAliases -Optional. Whether to include in the returned owners list aliases that are team aliases, e.g. Azure/azure-sdk-team - -.PARAMETER Test -Optional. Whether to run the script against hard-coded tests. - -#> -function Get-Codeowners( - [string] $TargetPath, - [string] $TargetDirectory, - [string] $ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool"), - [string] $DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", - [string] $ToolVersion = "1.0.0-dev.20230629.2", - [string] $VsoVariable = "", - [string] $CodeownersFileLocation = "", - [switch] $IncludeNonUserAliases - ) -{ - if ([string]::IsNullOrWhiteSpace($CodeownersFileLocation)) { - # The $PSScriptRoot is assumed to be azure-sdk-tools/eng/common/scripts/get-codeowners.ps1 - $CodeownersFileLocation = (Resolve-Path $PSScriptRoot/../../../.github/CODEOWNERS) - } - - # Backward compatibility: if $TargetPath is not provided, fall-back to the legacy $TargetDirectory - if ([string]::IsNullOrWhiteSpace($TargetPath)) { - $TargetPath = $TargetDirectory - } - if ([string]::IsNullOrWhiteSpace($TargetPath)) { - Write-Error "TargetPath (or TargetDirectory) parameter must be neither null nor whitespace." - return ,@() - } - - $jsonOutputFile = New-TemporaryFile - $codeownersToolCommand = Get-CodeownersTool -ToolPath $ToolPath -DevOpsFeed $DevOpsFeed -ToolVersion $ToolVersion - Write-Host "Executing: & $codeownersToolCommand --target-path $TargetPath --codeowners-file-path-or-url $CodeownersFileLocation --exclude-non-user-aliases:$(!$IncludeNonUserAliases) --owners-data-output-file $jsonOutputFile" - $commandOutput = & $codeownersToolCommand ` - --target-path $TargetPath ` - --codeowners-file-path-or-url $CodeownersFileLocation ` - --exclude-non-user-aliases:$(!$IncludeNonUserAliases) ` - --owners-data-output-file $jsonOutputFile ` - 2>&1 - - if ($LASTEXITCODE -ne 0) { - Write-Host "Command $codeownersToolCommand execution failed (exit code = $LASTEXITCODE). Output string: $commandOutput" - return ,@() - } else - { - Write-Host "Command $codeownersToolCommand executed successfully (exit code = 0). Command output string length: $($commandOutput.length)" - } - - # Assert: $commandOutput is a valid JSON representing: - # - a single CodeownersEntry, if the $TargetPath was a single path - # - or a dictionary of CodeownerEntries, keyes by each path resolved from a $TargetPath glob path. - # - # For implementation details, see Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main - - $fileContents = Get-Content $jsonOutputFile -Raw - $codeownersJson = ConvertFrom-Json -InputObject $fileContents - - if ($VsoVariable) { - $codeowners = $codeownersJson.Owners -join "," - Write-Host "##vso[task.setvariable variable=$VsoVariable;]$codeowners" - } - - return ,@($codeownersJson.Owners) -} \ No newline at end of file diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 deleted file mode 100644 index b40f9b4fa29..00000000000 --- a/eng/common/scripts/get-codeowners.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -<# -.SYNOPSIS -Please see the comment on Get-Codeowners defined in ./get-codeowners.lib.ps1 -#> -param ( - [string] $TargetPath = "", - [string] $TargetDirectory = "", - [string] $CodeownersFileLocation = "", - [switch] $IncludeNonUserAliases -) - -. $PSScriptRoot/get-codeowners.lib.ps1 - -return Get-Codeowners ` - -TargetPath $TargetPath ` - -TargetDirectory $TargetDirectory ` - -CodeownersFileLocation $CodeownersFileLocation ` - -IncludeNonUserAliases:$IncludeNonUserAliases \ No newline at end of file diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj deleted file mode 100644 index 8dbda797776..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - net6.0 - enable - Nullable - false - - - - - - - - - - - - - - - - - PreserveNewest - - - diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/CodeownersManualAnalysisTests.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/CodeownersManualAnalysisTests.cs deleted file mode 100644 index 1ce50113dad..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/CodeownersManualAnalysisTests.cs +++ /dev/null @@ -1,623 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.Json; -using Azure.Sdk.Tools.CodeOwnersParser; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests; - -/// -/// A class containing a set of tools, implemented as unit tests, -/// allowing you to view and diff owners of files of locally cloned repositories, -/// by obtaining the owners based on specified CODEOWNERS files. -/// -/// These tools are to be run manually, locally, by a developer. -/// They do not participate in an automated regression test suite. -/// -/// To run these tools, you will have to first manually comment out the "Ignore" -/// property of the "TestFixture" annotation below. -/// Then run the desired tool as a unit test, either from your IDE, -/// or "dotnet test" command line tool. -/// -/// These tools assume you have made local repo clones of relevant repositories. -/// Ensure that the local repo clones you run these tools against are clean. -/// This is because these tools do not support .gitignore. -/// Hence if you do local builds you might add minutes to runtime, and get spurious results. -/// -/// For explanation how to interpret and work with the output .csv file produced by these -/// tools, see comment on -/// -/// WriteOwnersDiffToCsv -/// -/// Related work: -/// Enable the new, regex-based, wildcard-supporting CODEOWNERS matcher -/// https://github.com/Azure/azure-sdk-tools/pull/5088 -/// -[TestFixture(Ignore = "Tools to be used manually")] -public class CodeownersManualAnalysisTests -{ - private const string OwnersDiffOutputPathSuffix = "_owners_diff.csv"; - private const string OwnersDataOutputPathSuffix = "_owners.csv"; - - /// - /// Given name of the language langName, returns path to a local clone of "azure-sdk-for-langName" - /// repository. - /// - /// This method assumes you have ensured the local clone is present at appropriate path ahead of time. - /// - /// - private static string LangRepoTargetDirPathSuffix(string langName) => "/../azure-sdk-for-" + langName; - - private const string CodeownersFilePathSuffix = "/.github/CODEOWNERS"; - - /// - /// This file is expected to be manually created by you, in your local repo clone. - /// For details of usage of this file, see: - /// - /// WriteTwoCodeownersFilesOwnersDiffToCsv - /// - /// - private const string SecondaryCodeownersFilePathSuffix = "/.github/CODEOWNERS2"; - - // Current dir, ".", is expected to be a dir in local clone of Azure/azure-sdk-tools repo, - // where "." denotes "/artifacts/bin/Azure.Sdk.Tools.CodeOwnersParser.Tests/Debug/net6.0". - private const string CurrentDir = "/artifacts/bin/Azure.Sdk.Tools.CodeOwnersParser.Tests/Debug/net6.0"; - - #region Tests - Owners data - - [Test] // Runtime <1s - public void OwnersForAzureDev() - => WriteOwnersToCsv( - targetDirPathSuffix: "/../azure-dev", - outputFileNamePrefix: "azure-dev", - ignoredPathPrefixes: ".git|artifacts"); - - // @formatter:off - [Test] public void OwnersForAzureSdkForAndroid() => WriteLangRepoOwnersToCsv("android"); // Runtime <1s - [Test] public void OwnersForAzureSdkForC() => WriteLangRepoOwnersToCsv("c"); // Runtime <1s - [Test] public void OwnersForAzureSdkForCpp() => WriteLangRepoOwnersToCsv("cpp"); // Runtime <1s - [Test] public void OwnersForAzureSdkForGo() => WriteLangRepoOwnersToCsv("go"); // Runtime <1s - [Test] public void OwnersForAzureSdkForIos() => WriteLangRepoOwnersToCsv("ios"); // Runtime <1s - [Test] public void OwnersForAzureSdkForJava() => WriteLangRepoOwnersToCsv("java"); // Runtime ~1m 11s - [Test] public void OwnersForAzureSdkForJs() => WriteLangRepoOwnersToCsv("js"); // Runtime ~1m 53s - [Test] public void OwnersForAzureSdkForNet() => WriteLangRepoOwnersToCsv("net"); // Runtime ~30s - [Test] public void OwnersForAzureSdkForPython() => WriteLangRepoOwnersToCsv("python"); // Runtime ~30s - // @formatter:on - - [Test] // Runtime <1s - public void OwnersForAzureSdkTools() - => WriteOwnersToCsv( - targetDirPathSuffix: "", - outputFileNamePrefix: "azure-sdk-tools", - ignoredPathPrefixes: ".git|artifacts"); - - #endregion - - #region Tests - Owners diffs for differing CODEOWNERS contents. - - [Test] // Runtime <1s - public void OwnersDiffForAzureDev() - => WriteTwoCodeownersFilesOwnersDiffToCsv( - targetDirPathSuffix: "/../azure-dev", - outputFileNamePrefix: "azure-dev", - ignoredPathPrefixes: ".git|artifacts"); - - // https://github.com/Azure/azure-sdk-for-android/blob/main/.github/CODEOWNERS - // No build failure notifications are configured for this repo. - // Runtime: <1s - [Test] public void OwnersDiffForAzureSdkForAndroid() => WriteLangRepoOwnersDiffToCsv("android"); - - // https://github.com/Azure/azure-sdk-for-c/blob/main/.github/CODEOWNERS - // Runtime: <1s - [Test] public void OwnersDiffForAzureSdkForC() => WriteLangRepoOwnersDiffToCsv("c"); - - // https://github.com/Azure/azure-sdk-for-cpp/blob/main/.github/CODEOWNERS - // Runtime: <1s - [Test] public void OwnersDiffForAzureSdkForCpp() => WriteLangRepoOwnersDiffToCsv("cpp"); - - // https://github.com/Azure/azure-sdk-for-go/blob/main/.github/CODEOWNERS - // Runtime: ~2s - [Test] public void OwnersDiffForAzureSdkForGo() => WriteLangRepoOwnersDiffToCsv("go"); - - // https://github.com/Azure/azure-sdk-for-ios/blob/main/.github/CODEOWNERS - // No build failure notifications are configured for this repo. - // Runtime: <1s - [Test] public void OwnersDiffForAzureSdkForIos() => WriteLangRepoOwnersDiffToCsv("ios"); - - // https://github.com/Azure/azure-sdk-for-java/blob/main/.github/CODEOWNERS - // Runtime: ~2m 32s - [Test] public void OwnersDiffForAzureSdkForJava() => WriteLangRepoOwnersDiffToCsv("java"); - - // https://github.com/Azure/azure-sdk-for-js/blob/main/.github/CODEOWNERS - // Runtime: ~3m 49s - [Test] public void OwnersDiffForAzureSdkForJs() => WriteLangRepoOwnersDiffToCsv("js"); - - // https://github.com/Azure/azure-sdk-for-net/blob/main/.github/CODEOWNERS - // Runtime: ~1m 01s - [Test] public void OwnersDiffForAzureSdkForNet() => WriteLangRepoOwnersDiffToCsv("net"); - - // https://github.com/Azure/azure-sdk-for-python/blob/main/.github/CODEOWNERS - // Runtime: ~45s - [Test] public void OwnersDiffForAzureSdkForPython() => WriteLangRepoOwnersDiffToCsv("python"); - - #endregion - - #region Parameterized tests - Owners - - private void WriteLangRepoOwnersToCsv(string langName) - => WriteOwnersToCsv( - targetDirPathSuffix: LangRepoTargetDirPathSuffix(langName), - outputFileNamePrefix: $"azure-sdk-for-{langName}", - ignoredPathPrefixes: ".git|artifacts"); - - private void WriteOwnersToCsv( - string targetDirPathSuffix, - string outputFileNamePrefix, - string ignoredPathPrefixes = Program.DefaultIgnoredPrefixes) - { - string rootDir = PathNavigatingToRootDir(CurrentDir); - string targetDir = rootDir + targetDirPathSuffix; - Debug.Assert(Directory.Exists(targetDir), - $"Ensure you have cloned the repo into '{targetDir}'. " + - "See comments on CodeownersManualAnalysisTests and WriteOwnersToCsv for details."); - Debug.Assert(File.Exists(targetDir + CodeownersFilePathSuffix), - $"Ensure you have cloned the repo into '{targetDir}'. " + - "See comments on CodeownersManualAnalysisTests and WriteOwnersToCsv for details."); - WriteOwnersToCsv( - targetDirPathSuffix, - CodeownersFilePathSuffix, - ignoredPathPrefixes, - outputFileNamePrefix); - } - - #endregion - - #region Parameterized tests - Owners diff - - private void WriteLangRepoOwnersDiffToCsv(string langName) - => WriteTwoCodeownersFilesOwnersDiffToCsv( - targetDirPathSuffix: LangRepoTargetDirPathSuffix(langName), - outputFileNamePrefix: $"azure-sdk-for-{langName}", - ignoredPathPrefixes: ".git|artifacts"); - - /// - /// This method is an invocation of: - /// - /// WriteOwnersDiffToCsv - /// - /// with following meanings bound to LEFT and RIGHT: - /// - /// LEFT: RetrieveCodeowners configuration given input local repository clone CODEOWNERS file. - /// - /// RIGHT: RetrieveCodeowners configuration given input repository CODEOWNERS2 file, - /// located beside CODEOWNERS file. - /// - /// The CODEOWNERS2 file is expected to be created manually by you. This way you can diff CODEOWNERS - /// to whatever version of it you want to express in CODEOWNERS2. For example, CODEOWNERS2 could have - /// contents of CODEOWNERS as seen in an open PR pending being merged. - /// - /// Note that modifying or reordering existing paths may always impact which PR reviewers are auto-assigned, - /// but the build failure notification recipients changes apply only to paths that represent - /// build definition .yml files. - /// - private void WriteTwoCodeownersFilesOwnersDiffToCsv( - string targetDirPathSuffix, - string outputFileNamePrefix, - string ignoredPathPrefixes = Program.DefaultIgnoredPrefixes) - { - string rootDir = PathNavigatingToRootDir(CurrentDir); - string targetDir = rootDir + targetDirPathSuffix; - Debug.Assert(Directory.Exists(targetDir), - $"Ensure you have cloned the repo into '{targetDir}'. " + - "See comments on CodeownersManualAnalysisTests and WriteTwoCodeownersFilesOwnersDiffToCsv for details."); - Debug.Assert(File.Exists(targetDir + CodeownersFilePathSuffix), - $"Ensure you have cloned the repo into '{targetDir}'. " + - "See comments on CodeownersManualAnalysisTests and WriteTwoCodeownersFilesOwnersDiffToCsv for details."); - Debug.Assert(File.Exists(targetDir + SecondaryCodeownersFilePathSuffix), - $"Ensure you have created '{Path.GetFullPath(targetDir + SecondaryCodeownersFilePathSuffix)}'. " + - $"See comment on WriteTwoCodeownersFilesOwnersDiffToCsv for details."); - - WriteOwnersDiffToCsv( - new[] - { - (targetDirPathSuffix, CodeownersFilePathSuffix, ignoredPathPrefixes), - (targetDirPathSuffix, SecondaryCodeownersFilePathSuffix, ignoredPathPrefixes) - }, - outputFileNamePrefix); - } - - #endregion - - #region private static - - /// - /// This method is similar to: - /// - /// WriteOwnersDiffToCsv - /// - /// Except it is not doing any diffing: it just evaluates one invocation of - /// Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main - /// and returns its information, in similar, but simplified table format. - /// - /// If given path, provided in column PATH, did not match any path in CODEOWNERS file, - /// the column PATH EXPRESSION will have a value of _____ . - /// - /// In addition, this method also does an validation of CODEOWNERS paths - /// and if it find a problem with given path, it returns output lines with ISSUE column - /// populated and PATH column empty, as there is no path to speak of - only CODEOWNERS path, - /// provided in PATH EXPRESSION column, is present. - /// - /// The ISSUE column has following codes: - /// - /// INVALID_PATH_CONTAINS_UNSUPPORTED_FRAGMENTS - /// All CODEOWNERS paths must not contain unsupported path fragments, as defined by: - /// Azure.Sdk.Tools.CodeOwnersParser.MatchedCodeownersEntry.ContainsUnsupportedFragments - /// - /// INVALID_PATH_SHOULD_START_WITH_SLASH - /// All CODEOWNERS paths must start with "/", but given path doesn't. - /// Such path will still be processed by our CODEOWNERS interpreter, but nevertheless it is - /// invalid and should be fixed. - /// - /// INVALID_PATH_MATCHES_DIR_EXACTLY - /// INVALID_PATH_MATCHES_DIR_EXACTLY_AND_NAME_PREFIX - /// INVALID_PATH_MATCHES_NAME_PREFIX - /// CODEOWNERS file contains a simple (i.e. without wildcards) path that is expected to match against - /// a file, as it does not end with "/". However, the repository contains one or more of the following: - /// - a directory with the same path - /// - a directory with such path being its name prefix: e.g. the path is /foobar and the dir is /foobarbaz/ - /// - a file with such path being its name prefix: e.g. the path is /foobar and the file is /foobarbaz.txt - /// - /// Such paths are invalid because they ambiguous and need to be disambiguated. - /// If the match is only to exact directory, then such CODEOWNERS path will never match any input path. - /// Usually the proper fix - /// is to add the missing suffix "/" to the path to make it correctly match against the existing directory. - /// If the match is to directory prefix, then this can be solved by appending "*/". This will match both - /// exact directories, and directory prefixes. - /// If the match is to file name prefix only, this can be fixed by appending "*". - /// If the match is both to directory and file name prefixes, possibly multiple paths need to be used, - /// one with "*/" suffix and one with "*" suffix. - /// - /// WILDCARD_FILE_PATH_NEEDS_MANUAL_EVAL - /// Same situation as above, but the CODEOWNERS path is a file path with a wildcard, hence current - /// validation implementation cannot yet determine if it should be a path to directory or not. - /// Hence, this needs to be checked manually by ensuring that the wildcard file path matches - /// at least one file in the repository. - /// - /// Known limitation: - /// If given CODEOWNERS path has no owners listed on its line, this method will not report such path as invalid. - /// - private static void WriteOwnersToCsv( - string targetDirPathSuffix, - string codeownersFilePathSuffix, - string ignoredPrefixes, - string outputFilePrefix) - { - var stopwatch = Stopwatch.StartNew(); - string rootDir = PathNavigatingToRootDir(CurrentDir); - string targetDir = rootDir + targetDirPathSuffix; - - Dictionary ownersData = RetrieveCodeowners( - targetDirPathSuffix, - codeownersFilePathSuffix, - ignoredPrefixes); - - List outputLines = - new List { "PATH | PATH EXPRESSION | OWNERS | ISSUE" }; - foreach (KeyValuePair kvp in ownersData) - { - string path = kvp.Key; - CodeownersEntry entry = kvp.Value; - outputLines.Add( - $"{path} " + - $"| {(entry.IsValid ? entry.PathExpression : "_____")} " + - $"| {string.Join(",", entry.Owners)}"); - } - - outputLines.AddRange(PathsWithIssues(targetDir, codeownersFilePathSuffix, paths: ownersData.Keys.ToArray())); - - var outputFilePath = outputFilePrefix + OwnersDataOutputPathSuffix; - File.WriteAllLines(outputFilePath, outputLines); - Console.WriteLine($"DONE writing out owners. " + - $"Output written out to {Path.GetFullPath(outputFilePath)}. " + - $"Time taken: {stopwatch.Elapsed}."); - } - - // Possible future work: - // instead of returning lines with issues, consider returning the modified & fixed CODEOWNERS file. - // It could work by reading all the lines, then replacing the wrong - // lines by using dict replacement. Need to be careful about retaining spaces to not misalign, - // e.g. - // "sdk/ @own1" --> "/sdk/ @own1" // space removed to keep alignment - // but also: - // "sdk/ @own1" --> "/sdk/ @own1" // space not removed, because it would be invalid. - private static List PathsWithIssues( - string targetDir, - string codeownersPathSuffix, - string[] paths) - { - List outputLines = new List(); - List entries = - CodeownersFile.GetCodeownersEntriesFromFileOrUrl(targetDir + codeownersPathSuffix) - .Where(entry => !entry.PathExpression.StartsWith("#")) - .ToList(); - - outputLines.AddRange(PathsWithMissingPrefixSlash(entries)); - outputLines.AddRange(PathsWithMissingSuffixSlash(targetDir, entries, paths)); - outputLines.AddRange(InvalidPaths(entries)); - // TODO: add a check here for CODEOWNERS paths that do not match any dir or file. - - return outputLines; - } - - private static List PathsWithMissingPrefixSlash(List entries) - => entries - .Where(entry => !entry.PathExpression.StartsWith("/")) - .Select(entry => - "|" + - $"{entry.PathExpression} " + - $"| {string.Join(",", entry.Owners)}" + - "| INVALID_PATH_SHOULD_START_WITH_SLASH") - .ToList(); - - private static List PathsWithMissingSuffixSlash( - string targetDir, - List entries, - string[] paths) - { - List outputLines = new List(); - foreach (CodeownersEntry entry in entries.Where(entry => !entry.PathExpression.EndsWith("/"))) - { - if (entry.ContainsWildcard) - { - // We do not support "the path is to file while it should be to directory" validation for paths - // with wildcards yet. To do that, we would first need to resolve the path and see if there exists - // a concrete path that includes the CODEOWNERS paths supposed-file-name as - // infix dir. - // For example, /a/**/b could match against /a/foo/b/c, meaning - // the path is invalid. - outputLines.Add( - "|" + - $"{entry.PathExpression} " + - $"| {string.Join(",", entry.Owners)}" + - "| WILDCARD_FILE_PATH_NEEDS_MANUAL_EVAL"); - } - else - { - string trimmedPathExpression = entry.PathExpression.TrimStart('/'); - - bool matchesDirExactly = MatchesDirExactly(targetDir, trimmedPathExpression); - bool matchesNamePrefix = MatchesNamePrefix(paths, trimmedPathExpression); - - if (matchesDirExactly || matchesNamePrefix) - { - string msgCode = matchesDirExactly && matchesNamePrefix ? "MATCHES_DIR_EXACTLY_AND_NAME_PREFIX" : - matchesDirExactly ? "MATCHES_DIR_EXACTLY" : "MATCHES_NAME_PREFIX"; - - outputLines.Add( - "|" + - $"{entry.PathExpression} " + - $"| {string.Join(",", entry.Owners)}" + - $"| INVALID_PATH_{msgCode}"); - } - } - } - return outputLines; - } - - private static bool MatchesNamePrefix(string[] paths, string trimmedPathExpression) - => paths.Any( - path => - { - string trimmedPath = path.TrimStart('/'); - bool pathIsChildDir = trimmedPath.Contains("/") - && trimmedPath.Length > trimmedPathExpression.Length - && trimmedPath.Substring(trimmedPathExpression.Length).StartsWith('/'); - return trimmedPath.StartsWith(trimmedPathExpression) - && trimmedPath.Length != trimmedPathExpression.Length - && !pathIsChildDir; - }); - - private static bool MatchesDirExactly(string targetDir, string trimmedPathExpression) - { - string pathToDir = Path.Combine( - targetDir, - trimmedPathExpression.Replace('/', Path.DirectorySeparatorChar)); - return Directory.Exists(pathToDir); - } - - private static List InvalidPaths(List entries) - => entries - .Where(entry => !MatchedCodeownersEntry.IsCodeownersPathValid(entry.PathExpression)) - .Select( - entry => - "|" + - $"{entry.PathExpression} " + - $"| {string.Join(",", entry.Owners)}" + - "| INVALID_PATH") - .ToList(); - - /// - /// Writes to .csv file the difference of owners for all paths in given repository, - /// between two invocations of Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main, - /// denoted as LEFT and RIGHT. RetrieveCodeOwners.Program.Main method reads - /// all files in given input repository, and tries to find owners for them based on - /// CODEOWNERS matching configuration given as its parameters. - /// - /// You can import the test output into Excel, using .csv import wizard and - /// selecting "|" as column separator. - /// - /// The resulting .csv file has following headers: - /// - /// DIFF CODE | PATH | LEFT PATH EXPRESSION | RIGHT PATH EXPRESSION | LEFT OWNERS | RIGHT OWNERS - /// - /// where LEFT denotes the RetrieveCodeOwners.Program.Main configuration as provided by input[0]. - /// and RIGHT denotes the RetrieveCodeOwners.Program.Main configuration as provided by input[1]. - /// - /// The columns have following values and meanings: - /// - /// DIFF CODE: - /// PATH _____-RIGHT - /// A file with given path, given in the column PATH, was not matched to any CODEOWNERS - /// path when using the LEFT configuration but it was matched when using the RIGHT configuration. - /// - /// PATH LEFT -_____ - /// Analogous to the case described above, but LEFT configuration has matched, and RIGHT didn't. - /// - /// PATH _____-_____ - /// A file with given path did not match to any CODEOWNERS path, whether using the LEFT - /// configuration or RIGHT configuration. - /// Such file has effectively no owners assigned, no matter which configuration is used. - /// - /// OWNERS DIFF - /// A file with given path matched both when using LEFT and RIGHT configurations, but - /// the CODEOWNERS path to which it matched has different set of owners. - /// - /// PATH: - /// A path to the file being matched against CODEOWNERS path to determine owners. - /// - /// LEFT PATH EXPRESSION: - /// RIGHT PATH EXPRESSION: - /// A CODEOWNERS path that matched to PATH when using LEFT (or RIGHT, respectively) configuration. - /// - /// LEFT OWNERS: - /// RIGHT OWNERS: - /// The owners assigned to given LEFT PATH EXPRESSION (or RIGHT PATH EXPRESSION, respectively). - /// - private static void WriteOwnersDiffToCsv( - ( - string targetDirPathSuffix, - string codeownersFilePathSuffix, - string ignoredPrefixes - )[] input, - string outputFilePrefix) - { - var stopwatch = Stopwatch.StartNew(); - - Dictionary leftOwners = RetrieveCodeowners( - input[0].targetDirPathSuffix, - input[0].codeownersFilePathSuffix, - input[0].ignoredPrefixes); - Dictionary rightOwners = RetrieveCodeowners( - input[1].targetDirPathSuffix, - input[1].codeownersFilePathSuffix, - input[1].ignoredPrefixes); - - string[] diffLines = PathOwnersDiff(leftOwners, rightOwners); - - var outputFilePath = outputFilePrefix + OwnersDiffOutputPathSuffix; - File.WriteAllLines(outputFilePath, diffLines); - Console.WriteLine($"DONE diffing. " + - $"Output written out to {Path.GetFullPath(outputFilePath)}. " + - $"Time taken: {stopwatch.Elapsed}."); - } - - private static Dictionary RetrieveCodeowners( - string targetDirPathSuffix, - string codeownersFilePathSuffixToRootDir, - string ignoredPathPrefixes) - { - string rootDir = PathNavigatingToRootDir(CurrentDir); - string targetDir = rootDir + targetDirPathSuffix; - string codeownersFilePath = targetDir + codeownersFilePathSuffixToRootDir; - Debug.Assert(Directory.Exists(targetDir)); - Debug.Assert(File.Exists(codeownersFilePath)); - - string actualOutput, actualErr; - int returnCode; - using (var consoleOutput = new ConsoleOutput()) - { - // Act - returnCode = Program.Main( - targetPath: "/**", - codeownersFilePath, - // false because we want to see the full owners diff, but observe that - // for the build failure notification recipients determination it should be true, - // because Contacts.GetMatchingCodeownersEntry() calls ExcludeNonUserAliases(). - excludeNonUserAliases: false, - targetDir, - ignoredPathPrefixes); - - actualOutput = consoleOutput.GetStdout(); - actualErr = consoleOutput.GetStderr(); - } - - var actualEntries = JsonSerializer.Deserialize>(actualOutput)!; - return actualEntries; - } - - private static string PathNavigatingToRootDir(string currentDir) - => "./" + string.Join( - "/", - currentDir - .Split("/", StringSplitOptions.RemoveEmptyEntries) - .Select(_ => "..")); - - private static string[] PathOwnersDiff( - Dictionary left, - Dictionary right) - { - Debug.Assert( - left.Keys.ToHashSet().SetEquals(right.Keys.ToHashSet()), - "The compared maps of owner data are expected to have the same paths (keys)."); - - List outputLines = new List - { - "DIFF CODE | PATH | LEFT PATH EXPRESSION | RIGHT PATH EXPRESSION | LEFT OWNERS | RIGHT OWNERS" - }; - foreach (string path in left.Keys) - { - if (left[path].IsValid && right[path].IsValid) - { - // Path matched against an entry in both "left" and "right" owners data. - // Here we determine if the owners lists match. - outputLines.AddRange(PathOwnersDiff(path, left[path], right[path])); - } - else if (left[path].IsValid && !right[path].IsValid) - { - // Path matched against an entry in the "left" owners data, but not in the right. - outputLines.Add($"PATH LEFT -_____ " + - $"| {path} | {left[path].PathExpression} | | {string.Join(",",left[path].Owners)} |"); - } - else if (!left[path].IsValid && right[path].IsValid) - { - // Path matched against an entry in the "right" owners data, but not in the right. - outputLines.Add($"PATH _____-RIGHT " + - $"| {path} | | {right[path].PathExpression} | | {string.Join(",",right[path].Owners)}"); - } - else - { - // Path did not match against any owners data, not in "left" nor "right". - outputLines.Add($"PATH _____-_____ | {path} |"); - } - } - - return outputLines.ToArray(); - } - - private static string[] PathOwnersDiff( - string path, - CodeownersEntry left, - CodeownersEntry right) - { - Debug.Assert(left.IsValid); - Debug.Assert(right.IsValid); - - List outputLines = new List(); - - if (!left.Owners.ToHashSet().SetEquals(right.Owners.ToHashSet())) - { - // Given path owners differ between "left" an "right" owners data. - outputLines.Add( - $"OWNERS DIFF | {path} " + - $"| {left.PathExpression} | {right.PathExpression} " + - $"| {string.Join(", ", left.Owners)} | {string.Join(", ", right.Owners)}"); - } - - return outputLines.ToArray(); - } - - #endregion -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs deleted file mode 100644 index 8b1803391a3..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/ConsoleOutput.cs +++ /dev/null @@ -1,70 +0,0 @@ -using System; -using System.IO; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests -{ - /// - /// The class is to redirect console STDOUT and STDERR to string writer. - /// - public class ConsoleOutput : IDisposable - { - private readonly StringWriter stdoutWriter, stderrWriter; - private readonly TextWriter originalStdout, originalStderr; - - /// - /// The constructor is where we take in the console output and output to string writer. - /// - public ConsoleOutput() - { - this.stdoutWriter = new StringWriter(); - this.stderrWriter = new StringWriter(); - this.originalStdout = Console.Out; - this.originalStderr = Console.Error; - Console.SetOut(this.stdoutWriter); - Console.SetError(this.stderrWriter); - } - - /// - /// Writes the text representation of a string builder to the string. - /// - /// The string from console output. - public string GetStdout() - => this.stdoutWriter.ToString(); - - public string[] GetStdoutLines() - => this.stdoutWriter.ToString().Split(Environment.NewLine); - - public string GetStderr() - => this.stderrWriter.ToString(); - - public string[] GetStderrLines() - => this.stderrWriter.ToString().Split(Environment.NewLine); - - /// - /// Releases all resources used by the originalOutput and stringWriter object. - /// - public void Dispose() - { - Console.SetOut(this.originalStdout); - Console.SetError(this.originalStderr); - this.stdoutWriter.Dispose(); - this.stderrWriter.Dispose(); - this.originalStdout.Dispose(); - this.originalStderr.Dispose(); - // https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca1816 - GC.SuppressFinalize(this); - } - - /// - /// Closes the current writer and releases any system resources associated with the writer - /// - public void Close() - { - this.stdoutWriter.Close(); - this.stderrWriter.Close(); - this.originalStderr.Close(); - this.originalStdout.Close(); - } - } - -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/RetrieveCodeOwnersProgramTests.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/RetrieveCodeOwnersProgramTests.cs deleted file mode 100644 index 1879eecda4d..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/RetrieveCodeOwnersProgramTests.cs +++ /dev/null @@ -1,214 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using Azure.Sdk.Tools.CodeOwnersParser; -using NUnit.Framework; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests; - -/// -/// Test class for Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main(), -/// -/// The tests assertion expectations are set to match GitHub CODEOWNERS interpreter behavior, -/// as explained here: -/// https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -/// and as observed based on manual tests and verification. -/// -/// For additional related tests, please see: -/// - Azure.Sdk.Tools.CodeOwnersParser.Tests.CodeownersFileTests -/// -[TestFixture] -public class RetrieveCodeOwnersProgramTests -{ - /// - /// A battery of test cases exercising the Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main executable. - /// - /// Each test case is composed of a targetPath and expected CodeownersEntry that is to match against - /// the targetPath when the executable is executed. - /// - /// These test battery is used in the following ways: - /// - /// 1. In OutputsCorrectCodeownersOnSimpleTargetPath parameterized unit test, each test case is exercised - /// by running the executable with targetPath provided as input targetPath. - /// - /// 2. In OutputsCorrectCodeownersOnGlobTargetPath the entire test battery is asserted against - /// by running the executable with targetPaths set to "/**", thus entering the glob-matching mode and finding - /// all the targetPaths present in this battery. - /// - /// Preconditions for running tests against this battery: - /// - directory "./TestData/InputDir" and file "./TestData/test_CODEOWNERS" contain appropriate contents - /// - the exercised executable is passed as input appropriate arguments pointing to the file system; - /// consult the aforementioned tests for concrete values. - /// - private static readonly TestCase[] testCases = - { - // @formatter:off - // targetPath expected CodeownersEntry - new ("a.txt" , new CodeownersEntry("/*", new List { "star" })), - new ("b.txt" , new CodeownersEntry("/*", new List { "star" })), - new ("foo/a.txt" , new CodeownersEntry("/foo/**/a.txt", new List { "foo_2star_a" })), - new ("foo/b.txt" , new CodeownersEntry("/**", new List { "2star" })), - new ("foo/bar/a.txt" , new CodeownersEntry("/foo/*/a.txt", new List { "foo_star_a_1", "foo_star_a_2" })), - new ("foo/bar/b.txt" , new CodeownersEntry("/**", new List { "2star" })), - new ("baz/cor/c.txt" , new CodeownersEntry("/baz*", new List { "baz_star" })), - new ("baz_.txt" , new CodeownersEntry("/baz*", new List { "baz_star" })), - new ("qux/abc/d.txt" , new CodeownersEntry("/qux/", new List { "qux" })), - new ("cor.txt" , new CodeownersEntry("/*", new List { "star" })), - new ("cor2/a.txt" , new CodeownersEntry("/**", new List { "2star" })), - new ("cor/gra/a.txt" , new CodeownersEntry("/**", new List { "2star" })) - // @formatter:on - }; - - private static Dictionary TestCasesAsDictionary - => testCases.ToDictionary( - testCase => testCase.TargetPath, - testCase => testCase.ExpectedCodeownersEntry); - - /// - /// Please see comment on RetrieveCodeOwnersProgramTests.testCases - /// - [TestCaseSource(nameof(testCases))] - public void OutputsCorrectCodeownersOnSimpleTargetPath(TestCase testCase) - { - const string targetDir = "./TestData/InputDir"; - const string codeownersFilePathOrUrl = "./TestData/test_CODEOWNERS"; - const bool excludeNonUserAliases = false; - - var targetPath = testCase.TargetPath; - var expectedEntry = testCase.ExpectedCodeownersEntry; - - // Act - (string actualOutput, string actualErr, int returnCode) = RunProgramMain( - targetPath, - codeownersFilePathOrUrl, - excludeNonUserAliases, - targetDir); - - CodeownersEntry actualEntry = TryDeserializeActualEntryFromSimpleTargetPath(actualOutput, actualErr); - - Assert.Multiple(() => - { - Assert.That(actualEntry, Is.EqualTo(expectedEntry), $"path: {targetPath}"); - Assert.That(returnCode, Is.EqualTo(0)); - Assert.That(actualErr, Is.EqualTo(string.Empty)); - }); - } - - /// - /// Please see comment on RetrieveCodeOwnersProgramTests.testCases - /// - [Test] - public void OutputsCorrectCodeownersOnGlobTargetPath() - { - const string targetDir = "./TestData/InputDir"; - const string targetPath = "/**"; - const string codeownersFilePathOrUrl = "./TestData/test_CODEOWNERS"; - const bool excludeNonUserAliases = false; - - Dictionary expectedEntriesByPath = TestCasesAsDictionary; - - // Act - (string actualOutput, string actualErr, int returnCode) = RunProgramMain( - targetPath, - codeownersFilePathOrUrl, - excludeNonUserAliases, - targetDir); - - Dictionary actualEntriesByPath = TryDeserializeActualEntriesFromGlobTargetPath(actualOutput, actualErr); - - Assert.Multiple(() => - { - AssertEntries(actualEntriesByPath, expectedEntriesByPath); - Assert.That(returnCode, Is.EqualTo(0)); - Assert.That(actualErr, Is.EqualTo(string.Empty)); - }); - } - - private static (string actualOutput, string actualErr, int returnCode) RunProgramMain( - string targetPath, - string codeownersFilePathOrUrl, - bool excludeNonUserAliases, - string targetDir) - { - string actualOutput, actualErr; - int returnCode; - using (var consoleOutput = new ConsoleOutput()) - { - // Act - returnCode = Program.Main( - targetPath, - codeownersFilePathOrUrl, - excludeNonUserAliases, - targetDir); - - actualOutput = consoleOutput.GetStdout(); - actualErr = consoleOutput.GetStderr(); - } - - return (actualOutput, actualErr, returnCode); - } - - private static CodeownersEntry TryDeserializeActualEntryFromSimpleTargetPath( - string actualOutput, - string actualErr) - { - CodeownersEntry actualEntry; - try - { - actualEntry = - JsonSerializer.Deserialize(actualOutput)!; - } - catch (Exception e) - { - Console.WriteLine(e); - Console.WriteLine("actualOutput: " + actualOutput); - Console.WriteLine("actualErr: " + actualErr); - throw; - } - - return actualEntry; - } - - private static Dictionary TryDeserializeActualEntriesFromGlobTargetPath( - string actualOutput, - string actualErr) - { - Dictionary actualEntries; - try - { - actualEntries = - JsonSerializer.Deserialize>(actualOutput)!; - } - catch (Exception e) - { - Console.WriteLine(e); - Console.WriteLine("actualOutput: " + actualOutput); - Console.WriteLine("actualErr: " + actualErr); - throw; - } - - return actualEntries; - } - - private static void AssertEntries( - Dictionary actualEntries, - Dictionary expectedEntries) - { - foreach (KeyValuePair kvp in actualEntries) - { - string path = kvp.Key; - CodeownersEntry actualEntry = kvp.Value; - Assert.That(actualEntry, Is.EqualTo(expectedEntries[path]), $"path: {path}"); - } - - Assert.That(actualEntries, Has.Count.EqualTo(expectedEntries.Count)); - } - - /// - /// Please see comment on RetrieveCodeOwnersProgramTests.testCases - /// - public record TestCase( - string TargetPath, - CodeownersEntry ExpectedCodeownersEntry); -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/b.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/b.txt deleted file mode 100644 index 30b72c114c7..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/b.txt +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests.TestData -{ - class b - { - } -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz/cor/c.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz/cor/c.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz_.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/baz_.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor/gra/a.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor/gra/a.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor2/a.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/cor2/a.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/a.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/a.txt deleted file mode 100644 index 0b1b706b30d..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/a.txt +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests.TestData.foo -{ - class a - { - } -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/b.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/b.txt deleted file mode 100644 index e1bac95f81c..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/b.txt +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners.Tests.TestData.foo -{ - class b - { - } -} diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/a.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/a.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/b.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/foo/bar/b.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/qux/abc/d.txt b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/qux/abc/d.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS deleted file mode 100644 index fdf41242c53..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS +++ /dev/null @@ -1,16 +0,0 @@ -# This file is a test resoure for the test: -# -# Azure.Sdk.Tools.RetrieveCodeOwners.Tests.WildcardTests.TestWildcard -# - -/** @2star -/* @star - -/foo/**/a.txt @foo_2star_a -/foo/*/a.txt @foo_star_a_1 @foo_star_a_2 - -/baz* @baz_star - -/qux/ @qux - -/cor @cor \ No newline at end of file diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Azure.Sdk.Tools.RetrieveCodeOwners.csproj b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Azure.Sdk.Tools.RetrieveCodeOwners.csproj deleted file mode 100644 index 6ac4695ec59..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Azure.Sdk.Tools.RetrieveCodeOwners.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - Exe - net6.0 - enable - Nullable - true - retrieve-codeowners - 1.0.0 - - - - - - - - - - - diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs deleted file mode 100644 index 07ca20b85b9..00000000000 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.Json; -using Azure.Sdk.Tools.CodeOwnersParser; - -namespace Azure.Sdk.Tools.RetrieveCodeOwners; - -/// -/// See Program.Main comment. -/// -public static class Program -{ - /// - /// See comment on Azure.Sdk.Tools.RetrieveCodeOwners.Program.Main, - /// for parameter "ignoredPathPrefixes". - /// - public const string DefaultIgnoredPrefixes = ".git"; - - /// - /// Given targetPath and CODEOWNERS file path or https url codeownersFilePathOrUrl, - /// prints out to stdout owners of the targetPath as determined by the CODEOWNERS data. - /// - /// The path whose owners are to be determined. Can be a glob path. - /// The https url or path to the CODEOWNERS file. - /// Whether owners that aren't users should be excluded from - /// the returned owners. - /// - /// The directory to search for file paths in case targetPath is a glob path. Unused otherwise. - /// - /// - /// A list of path prefixes, separated by |, to ignore when doing - /// glob-matching against glob targetPath. - /// Applies only if targetPath is a glob path. Unused otherwise. - /// Defaults to ".git". - /// Example usage: ".git|foo|bar" - /// - /// Override for the default URI where the team/storage blob data resides. - /// File to output the owners data to, will overwrite if the file exist. - /// - /// On STDOUT: The JSON representation of the matched CodeownersEntry. - /// "new CodeownersEntry()" if no path in the CODEOWNERS data matches. - ///

- /// From the Main method: exit code. 0 if successful, 1 if error. - /// - public static int Main( - string targetPath, - string codeownersFilePathOrUrl, - bool excludeNonUserAliases = false, - string? targetDir = null, - string ignoredPathPrefixes = DefaultIgnoredPrefixes, - string? teamStorageURI = null, - string? ownersDataOutputFile = null) - { - try - { - Trace.Assert(!string.IsNullOrWhiteSpace(targetPath)); - - targetPath = targetPath.Trim(); - targetDir = targetDir?.Trim(); - codeownersFilePathOrUrl = codeownersFilePathOrUrl.Trim(); - - Trace.Assert(!string.IsNullOrWhiteSpace(codeownersFilePathOrUrl)); - Trace.Assert(!targetPath.IsGlobFilePath() - || (targetDir != null && Directory.Exists(targetDir))); - - // The "object" here is effectively an union of two types: T1 | T2, - // where T1 is the type returned by GetCodeownersForGlobPath - // and T2 is the type returned by GetCodeownersForSimplePath. - object codeownersData = targetPath.IsGlobFilePath() - ? GetCodeownersForGlobPath( - new GlobFilePath(targetPath), - targetDir!, - codeownersFilePathOrUrl, - excludeNonUserAliases, - SplitIgnoredPathPrefixes(), - teamStorageURI) - : GetCodeownersForSimplePath( - targetPath, - codeownersFilePathOrUrl, - excludeNonUserAliases, - teamStorageURI); - - string codeownersJson = JsonSerializer.Serialize( - codeownersData, - new JsonSerializerOptions { WriteIndented = true }); - - Console.WriteLine(codeownersJson); - - // If the output data file is specified, write the json to that. - if (!string.IsNullOrEmpty(ownersDataOutputFile)) - { - // False in the ctor is to overwrite, not append - using (StreamWriter outputFile = new StreamWriter(ownersDataOutputFile, false)) - { - outputFile.WriteLine(codeownersJson); - } - } - return 0; - - string[] SplitIgnoredPathPrefixes() - => ignoredPathPrefixes.Split( - "|", - StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - } - catch (Exception e) - { - Console.Error.WriteLine(e); - return 1; - } - } - - private static Dictionary GetCodeownersForGlobPath( - GlobFilePath targetPath, - string targetDir, - string codeownersFilePathOrUrl, - bool excludeNonUserAliases, - string[]? ignoredPathPrefixes = null, - string? teamStorageURI=null) - { - ignoredPathPrefixes ??= Array.Empty(); - - Dictionary codeownersEntries = - CodeownersFile.GetMatchingCodeownersEntries( - targetPath, - targetDir, - codeownersFilePathOrUrl, - ignoredPathPrefixes, - teamStorageURI); - - if (excludeNonUserAliases) - codeownersEntries.Values.ToList().ForEach(entry => entry.ExcludeNonUserAliases()); - - return codeownersEntries; - } - - private static CodeownersEntry GetCodeownersForSimplePath( - string targetPath, - string codeownersFilePathOrUrl, - bool excludeNonUserAliases, - string? teamStorageURI = null) - { - CodeownersEntry codeownersEntry = - CodeownersFile.GetMatchingCodeownersEntry( - targetPath, - codeownersFilePathOrUrl, - teamStorageURI); - - if (excludeNonUserAliases) - codeownersEntry.ExcludeNonUserAliases(); - - return codeownersEntry; - } -} diff --git a/tools/code-owners-parser/CodeOwnersParser.sln b/tools/code-owners-parser/CodeOwnersParser.sln index 8fa63a1ca1c..a6c98e6be24 100644 --- a/tools/code-owners-parser/CodeOwnersParser.sln +++ b/tools/code-owners-parser/CodeOwnersParser.sln @@ -1,51 +1,31 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{55D665BF-A4B3-45EA-A2A0-B33AFB208766}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners", "Azure.Sdk.Tools.RetrieveCodeOwners\Azure.Sdk.Tools.RetrieveCodeOwners.csproj", "{FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners.Tests", "Azure.Sdk.Tools.RetrieveCodeOwners.Tests\Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj", "{798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EF585CCA-55F8-44EB-921A-30996CBAFC49}" - ProjectSection(SolutionItems) = preProject - ci.yml = ci.yml - ..\..\eng\common\scripts\get-codeowners.lib.ps1 = ..\..\eng\common\scripts\get-codeowners.lib.ps1 - ..\..\eng\common\scripts\get-codeowners.ps1 = ..\..\eng\common\scripts\get-codeowners.ps1 - ..\..\eng\common-tests\get-codeowners\get-codeowners.tests.ps1 = ..\..\eng\common-tests\get-codeowners\get-codeowners.tests.ps1 - EndProjectSection -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Debug|Any CPU.Build.0 = Debug|Any CPU - {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Release|Any CPU.ActiveCfg = Release|Any CPU - {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Release|Any CPU.Build.0 = Release|Any CPU - {FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FE65F92D-C71B-4E38-A4B2-3089EA7C5FEC}.Release|Any CPU.Build.0 = Release|Any CPU - {798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {798B8CAC-68FC-49FD-A0F6-51C0DC4A4D1D}.Release|Any CPU.Build.0 = Release|Any CPU - {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.Build.0 = Debug|Any CPU - {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.ActiveCfg = Release|Any CPU - {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {E32EEF98-9184-4346-8801-C5A8A1C7FD7D} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{55D665BF-A4B3-45EA-A2A0-B33AFB208766}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55D665BF-A4B3-45EA-A2A0-B33AFB208766}.Release|Any CPU.Build.0 = Release|Any CPU + {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.Build.0 = Debug|Any CPU + {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.ActiveCfg = Release|Any CPU + {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {E32EEF98-9184-4346-8801-C5A8A1C7FD7D} + EndGlobalSection +EndGlobal diff --git a/tools/code-owners-parser/ci.yml b/tools/code-owners-parser/ci.yml deleted file mode 100644 index d28af6b9058..00000000000 --- a/tools/code-owners-parser/ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. -trigger: - branches: - include: - - main - - feature/* - - release/* - - hotfix/* - paths: - include: - - tools/code-owners-parser - -pr: - branches: - include: - - main - - feature/* - - release/* - - hotfix/* - paths: - include: - - tools/code-owners-parser - - eng/common/scripts/get-codeowners - -extends: - template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml - parameters: - ToolDirectory: tools/code-owners-parser - TestPostSteps: - - template: /eng/common/pipelines/templates/steps/run-pester-tests.yml - parameters: - TargetDirectory: eng/common-tests/get-codeowners - TargetTags: UnitTest \ No newline at end of file diff --git a/tools/notification-configuration/notification-configuration.sln b/tools/notification-configuration/notification-configuration.sln index 85293118f18..51eddfbb447 100644 --- a/tools/notification-configuration/notification-configuration.sln +++ b/tools/notification-configuration/notification-configuration.sln @@ -1,67 +1,55 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.33103.201 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration", "notification-creator\Azure.Sdk.Tools.NotificationConfiguration.csproj", "{5759063D-A7B3-4D36-ACF4-5595C2789D27}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration.Tests", "notification-creator.Tests\Azure.Sdk.Tools.NotificationConfiguration.Tests.csproj", "{3097CBB4-ED3C-4273-AC67-F5D189CB94BA}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "..\code-owners-parser\CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{A9826C8B-85DF-48DB-8A05-40FB04833C42}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "..\code-owners-parser\Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{2146E1FF-04D1-4B19-9767-C011A73CB40D}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "..\identity-resolution\identity-resolution.csproj", "{9805B503-5469-412C-9A0C-F09F504F0ED8}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners", "..\code-owners-parser\Azure.Sdk.Tools.RetrieveCodeOwners\Azure.Sdk.Tools.RetrieveCodeOwners.csproj", "{3E5237F2-6536-4329-A9CF-92E42B040612}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.RetrieveCodeOwners.Tests", "..\code-owners-parser\Azure.Sdk.Tools.RetrieveCodeOwners.Tests\Azure.Sdk.Tools.RetrieveCodeOwners.Tests.csproj", "{8DAEC12F-8390-4122-9959-9CF3391F18CC}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EBC153AF-0244-4DFB-8084-E6C0ACAA5CF3}" - ProjectSection(SolutionItems) = preProject - ci.yml = ci.yml - README.md = README.md - EndProjectSection -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Release|Any CPU.Build.0 = Release|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.Build.0 = Release|Any CPU - {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.Build.0 = Release|Any CPU - {8DAEC12F-8390-4122-9959-9CF3391F18CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8DAEC12F-8390-4122-9959-9CF3391F18CC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8DAEC12F-8390-4122-9959-9CF3391F18CC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8DAEC12F-8390-4122-9959-9CF3391F18CC}.Release|Any CPU.Build.0 = Release|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.Build.0 = Release|Any CPU - {3E5237F2-6536-4329-A9CF-92E42B040612}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3E5237F2-6536-4329-A9CF-92E42B040612}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3E5237F2-6536-4329-A9CF-92E42B040612}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3E5237F2-6536-4329-A9CF-92E42B040612}.Release|Any CPU.Build.0 = Release|Any CPU - {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {1B352588-04B2-4983-B0F6-A559065D99EC} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.33103.201 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration", "notification-creator\Azure.Sdk.Tools.NotificationConfiguration.csproj", "{5759063D-A7B3-4D36-ACF4-5595C2789D27}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration.Tests", "notification-creator.Tests\Azure.Sdk.Tools.NotificationConfiguration.Tests.csproj", "{3097CBB4-ED3C-4273-AC67-F5D189CB94BA}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "..\code-owners-parser\CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{A9826C8B-85DF-48DB-8A05-40FB04833C42}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "..\code-owners-parser\Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{2146E1FF-04D1-4B19-9767-C011A73CB40D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "..\identity-resolution\identity-resolution.csproj", "{9805B503-5469-412C-9A0C-F09F504F0ED8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EBC153AF-0244-4DFB-8084-E6C0ACAA5CF3}" + ProjectSection(SolutionItems) = preProject + ci.yml = ci.yml + README.md = README.md + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5759063D-A7B3-4D36-ACF4-5595C2789D27}.Release|Any CPU.Build.0 = Release|Any CPU + {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.Build.0 = Release|Any CPU + {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.Build.0 = Release|Any CPU + {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.Build.0 = Release|Any CPU + {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {1B352588-04B2-4983-B0F6-A559065D99EC} + EndGlobalSection +EndGlobal From 8e6561b01ed0568de785d8c54b0db9eaa4a329f3 Mon Sep 17 00:00:00 2001 From: Scott Addie <10702007+scottaddie@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:03:18 -0500 Subject: [PATCH 47/93] Remove unused Monitor labels from CSV (#7038) --- tools/github/data/common-labels.csv | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tools/github/data/common-labels.csv b/tools/github/data/common-labels.csv index 7b2a309ae76..2ae80513d01 100644 --- a/tools/github/data/common-labels.csv +++ b/tools/github/data/common-labels.csv @@ -150,19 +150,9 @@ Mgmt-EngSys,Engineering System (Management Plane Specific),ffeb77 Migrate,,e99695 Mixed Reality,,e99695 Mobile Engagement,,e99695 -Monitor,"Monitor, Operational Insights",e99695 -Monitor - ActionGroups,,e99695 -Monitor - ActivityLogs,,e99695 -Monitor - Alerts,,e99695 +Monitor,"Monitor, Monitor Ingestion, Monitor Query",e99695 Monitor - ApplicationInsights,,e99695 -Monitor - Autoscale,,e99695 -Monitor - Diagnostic Settings,,e99695 Monitor - Exporter,Monitor OpenTelemetry Exporter,e99695 -Monitor - Log,Monitor Log Analytics,e99695 -Monitor - Log Analytics,,e99695 -Monitor - Metrics,,e99695 -Monitor - Operational Insights,,e99695 -Monitor - Query,,e99695 MySQL,,e99695 Network,,e99695 Network - Application Gateway,,e99695 From 1ab3b892d3ffa56c98bf06ab47e27edc71085367 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 3 Oct 2023 18:07:32 -0400 Subject: [PATCH 48/93] Use AzureLinux for stress cluster nodes (#7039) --- tools/stress-cluster/cluster/azure/cluster/cluster.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep index 8106327adc5..876bf129ebb 100644 --- a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep +++ b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep @@ -17,14 +17,14 @@ var kubernetesVersion = '1.26.6' var nodeResourceGroup = 'rg-nodes-${dnsPrefix}-${clusterName}-${groupSuffix}' var systemAgentPool = { - name: 'system' + name: 'systemal' count: 1 minCount: 1 maxCount: 4 mode: 'System' vmSize: 'Standard_D4ds_v4' type: 'VirtualMachineScaleSets' - osType: 'Linux' + osType: 'AzureLinux' enableAutoScaling: true enableEncryptionAtHost: true nodeLabels: { @@ -33,14 +33,14 @@ var systemAgentPool = { } var defaultAgentPool = { - name: 'default' + name: 'defaultal' count: defaultAgentPoolMinNodes minCount: defaultAgentPoolMinNodes maxCount: defaultAgentPoolMaxNodes mode: 'User' vmSize: 'Standard_D8a_v4' type: 'VirtualMachineScaleSets' - osType: 'Linux' + osType: 'AzureLinux' osDiskType: 'Ephemeral' enableAutoScaling: true enableEncryptionAtHost: true From d6ca8729491e1630e7fe3e1ebff19843cb3a4c8a Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Thu, 5 Oct 2023 12:19:13 -0700 Subject: [PATCH 49/93] [githubio-linkcheck] Reduce timeout from infinite to 6 hours (#7068) - Pipeline normally runs in 3 hours, so 6 hours is plenty of time - We should not use infinite timeouts, to avoid blocking build agents indefinitely --- eng/pipelines/githubio-linkcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/githubio-linkcheck.yml b/eng/pipelines/githubio-linkcheck.yml index f945543c74e..a19d4aaca24 100644 --- a/eng/pipelines/githubio-linkcheck.yml +++ b/eng/pipelines/githubio-linkcheck.yml @@ -11,7 +11,7 @@ variables: jobs: - job: CheckLinks displayName: Check and Cache Links - timeoutInMinutes: 0 + timeoutInMinutes: 360 steps: - task: PowerShell@2 displayName: 'azure-sdk link check' From 2ae7fb0cad3db037d19d8dea6b29ea37205e9ccb Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Thu, 5 Oct 2023 12:27:40 -0700 Subject: [PATCH 50/93] [live-test-cleanup] Reduce timeout from infinite to default (60 minutes) (#7069) - Pipeline normally runs in 15 minutes, so 60 minutes is plenty of time - We should not use infinite timeouts, to avoid blocking build agents indefinitely --- eng/pipelines/live-test-cleanup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml index 6b86821bca8..ec015618fad 100644 --- a/eng/pipelines/live-test-cleanup.yml +++ b/eng/pipelines/live-test-cleanup.yml @@ -62,7 +62,6 @@ stages: jobs: - job: Run - timeoutInMinutes: 0 pool: name: azsdk-pool-mms-ubuntu-2204-general vmImage: ubuntu-22.04 From fb08b61d9e2c94f167221039ccb01b6fa2fcd454 Mon Sep 17 00:00:00 2001 From: Jesse Squire Date: Fri, 6 Oct 2023 08:21:40 -0700 Subject: [PATCH 51/93] [JimBot] Update package version in use (#7071) * [JimBot] Update package version in use The focus of these changes is to update the version of the package used by Actions to deploy the changes recently made to remove the "CXP Attention" rule and tweak the "needs-team-triage" criteria to ignore issues with an assignment already made. * Updating package version for configured workflows --- .github/workflows/event-processor.yml | 2 +- .github/workflows/scheduled-event-processor.yml | 2 +- .../YmlAndConfigFiles/event-processor.yml | 2 +- .../YmlAndConfigFiles/scheduled-event-processor.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/event-processor.yml b/.github/workflows/event-processor.yml index c1b818d7d90..56845c175cd 100644 --- a/.github/workflows/event-processor.yml +++ b/.github/workflows/event-processor.yml @@ -55,7 +55,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230713.2 + --version 1.0.0-dev.20230929.3 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash diff --git a/.github/workflows/scheduled-event-processor.yml b/.github/workflows/scheduled-event-processor.yml index 547a5ce2ca0..d8cc88325dd 100644 --- a/.github/workflows/scheduled-event-processor.yml +++ b/.github/workflows/scheduled-event-processor.yml @@ -34,7 +34,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230713.2 + --version 1.0.0-dev.20230929.3 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash diff --git a/tools/github-event-processor/YmlAndConfigFiles/event-processor.yml b/tools/github-event-processor/YmlAndConfigFiles/event-processor.yml index 82b856eea0b..e9d80468052 100644 --- a/tools/github-event-processor/YmlAndConfigFiles/event-processor.yml +++ b/tools/github-event-processor/YmlAndConfigFiles/event-processor.yml @@ -59,7 +59,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230313.4 + --version 1.0.0-dev.20230929.3 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global working-directory: .github/workflows diff --git a/tools/github-event-processor/YmlAndConfigFiles/scheduled-event-processor.yml b/tools/github-event-processor/YmlAndConfigFiles/scheduled-event-processor.yml index d05cc1f6adc..87f420c70ae 100644 --- a/tools/github-event-processor/YmlAndConfigFiles/scheduled-event-processor.yml +++ b/tools/github-event-processor/YmlAndConfigFiles/scheduled-event-processor.yml @@ -36,7 +36,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230313.4 + --version 1.0.0-dev.20230929.3 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global working-directory: .github/workflows From cbd469a5c1230e776371f0d3e71870cb837e5408 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Fri, 6 Oct 2023 12:45:58 -0700 Subject: [PATCH 52/93] Detect all file diff types for eng/common changes (#7072) Fixes https://github.com/Azure/azure-sdk-tools/issues/5882 We need to set the difffilter to empty instead of the default of exclude deleted files when we are trying to verify there are no changes under eng/common. See test PR https://github.com/Azure/azure-sdk-for-python/pull/32348 which demonstrates us not detecting a deleted file under eng/common. I'll use that same test PR to verify this now catches that issue. --- .../templates/steps/eng-common-workflow-enforcer.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml b/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml index af8b009b582..b455fec2100 100644 --- a/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml +++ b/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml @@ -11,7 +11,7 @@ steps: if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-eng/common")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$") { - $filesInCommonDir = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath 'eng/common/*' + $filesInCommonDir = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath 'eng/common/*' -DiffFilterType "" if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0)) { Write-Host "##vso[task.LogIssue type=error;]Changes to files under 'eng/common' directory should not be made in this Repo`n${filesInCommonDir}" @@ -21,7 +21,7 @@ steps: } if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-.github/workflows")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$") { - $filesInCommonDir = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*' + $filesInCommonDir = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*' -DiffFilterType "" if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0)) { Write-Host "##vso[task.LogIssue type=error;]Changes to files under '.github/workflows' directory should not be made in this Repo`n${filesInCommonDir}" @@ -30,4 +30,4 @@ steps: } } displayName: Prevent changes to eng/common and .github/workflows outside of azure-sdk-tools repo - condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr'))) \ No newline at end of file + condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr'))) From 865b8619677f255e5d2d98b91ee494a41e780f34 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Mon, 9 Oct 2023 16:19:19 -0500 Subject: [PATCH 53/93] Create tool tsp-client (#6843) * create tool tsp-client * [tsp-client] Refactor tsp-client (#6887) * update options * add additional fs helper methods * update options and usage text * refactor + update commands * add additional directories support * add function to compile from shell * use additional dirs, call tsp compile * return empty list for additional dirs * add init option * use service as project name * improve directory naming * allow raw github content url for init, include additional dirs in yaml * rename file * use cp with promises * fix remove dir * remove existssync * clean up * more clean up * rename to syncAndGenerate * remove then from fs * remove then use * remove var * improve loop * remove shell usage * refactor url parser * refactor to use compile method from typespec lib * Rename function * add more init functionality * update log message * add support for local spec repo * add emitter options flag * reorganize code * support commit and repo update * support config url update * simplify emitter search in emitter-package.json * update package.json * prompt user for correct output dir * improve error message * clean up * add initial local spec support * fix author * update chaining format * fix resolveTspConfigUrl * simplify tspconfig variable search * fix getEmitterFromRepoConfig * fs updates * update package.json --------- Co-authored-by: catalinaperalta --- tools/tsp-client/.gitignore | 5 + tools/tsp-client/.prettierrc.json | 9 + tools/tsp-client/README.md | 46 ++++ tools/tsp-client/cmd/tsp-client.js | 2 + tools/tsp-client/package.json | 55 +++++ tools/tsp-client/src/fileTree.ts | 79 +++++++ tools/tsp-client/src/fs.ts | 92 ++++++++ tools/tsp-client/src/git.ts | 82 +++++++ tools/tsp-client/src/index.ts | 275 +++++++++++++++++++++++ tools/tsp-client/src/languageSettings.ts | 61 +++++ tools/tsp-client/src/log.ts | 86 +++++++ tools/tsp-client/src/network.ts | 92 ++++++++ tools/tsp-client/src/npm.ts | 51 +++++ tools/tsp-client/src/options.ts | 168 ++++++++++++++ tools/tsp-client/src/typespec.ts | 150 +++++++++++++ tools/tsp-client/test/fileTree.spec.ts | 21 ++ tools/tsp-client/test/network.spec.ts | 13 ++ tools/tsp-client/tsconfig.json | 23 ++ 18 files changed, 1310 insertions(+) create mode 100644 tools/tsp-client/.gitignore create mode 100644 tools/tsp-client/.prettierrc.json create mode 100644 tools/tsp-client/README.md create mode 100644 tools/tsp-client/cmd/tsp-client.js create mode 100644 tools/tsp-client/package.json create mode 100644 tools/tsp-client/src/fileTree.ts create mode 100644 tools/tsp-client/src/fs.ts create mode 100644 tools/tsp-client/src/git.ts create mode 100644 tools/tsp-client/src/index.ts create mode 100644 tools/tsp-client/src/languageSettings.ts create mode 100644 tools/tsp-client/src/log.ts create mode 100644 tools/tsp-client/src/network.ts create mode 100644 tools/tsp-client/src/npm.ts create mode 100644 tools/tsp-client/src/options.ts create mode 100644 tools/tsp-client/src/typespec.ts create mode 100644 tools/tsp-client/test/fileTree.spec.ts create mode 100644 tools/tsp-client/test/network.spec.ts create mode 100644 tools/tsp-client/tsconfig.json diff --git a/tools/tsp-client/.gitignore b/tools/tsp-client/.gitignore new file mode 100644 index 00000000000..fea69a2999f --- /dev/null +++ b/tools/tsp-client/.gitignore @@ -0,0 +1,5 @@ +node_modules +*.env +dist +types +temp diff --git a/tools/tsp-client/.prettierrc.json b/tools/tsp-client/.prettierrc.json new file mode 100644 index 00000000000..405804bb2ef --- /dev/null +++ b/tools/tsp-client/.prettierrc.json @@ -0,0 +1,9 @@ +{ + "arrowParens": "always", + "bracketSpacing": true, + "endOfLine": "lf", + "printWidth": 100, + "semi": true, + "singleQuote": false, + "tabWidth": 2 +} diff --git a/tools/tsp-client/README.md b/tools/tsp-client/README.md new file mode 100644 index 00000000000..b694fc03f10 --- /dev/null +++ b/tools/tsp-client/README.md @@ -0,0 +1,46 @@ +# tsp-client + +A simple command line tool for generating TypeSpec clients. + +### Usage +``` +tsp-client [options] +``` + +## Options + +### (required) + +The only positional parameter. Specifies the directory to pass to the language emitter. + +### --emitter, -e (required) + +Specifies which language emitter to use. Current choices are "csharp", "java", "javascript", "python", "openapi". + +Aliases are also available, such as cs, js, py, and ts. + +### --mainFile, -m + +Used when specifying a URL to a TSP file directly. Not required if using a `tsp-location.yaml` + +### --debug, -d + +Enables verbose debug logging to the console. + +### --no-cleanup + +Disables automatic cleanup of the temporary directory where the TSP is written and referenced npm modules are installed. + +## Examples + +Generating from a TSP file to a particular directory: + +``` +tsp-client -e openapi -m https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/cognitiveservices/OpenAI.Inference/main.tsp ./temp +``` + +Generating in a directory that contains a `tsp-location.yaml`: + +``` +tsp-client sdk/openai/openai +``` diff --git a/tools/tsp-client/cmd/tsp-client.js b/tools/tsp-client/cmd/tsp-client.js new file mode 100644 index 00000000000..ad945e82793 --- /dev/null +++ b/tools/tsp-client/cmd/tsp-client.js @@ -0,0 +1,2 @@ +#!/usr/bin/env node +await import("../dist/index.js"); diff --git a/tools/tsp-client/package.json b/tools/tsp-client/package.json new file mode 100644 index 00000000000..fdc74426feb --- /dev/null +++ b/tools/tsp-client/package.json @@ -0,0 +1,55 @@ +{ + "name": "@azure-tools/tsp-client", + "version": "0.0.1", + "private": "true", + "description": "A tool to generate Azure SDKs from TypeSpec", + "main": "dist/index.js", + "scripts": { + "build": "npm run clean && npm run build:tsc", + "build:tsc": "tsc", + "clean": "rimraf ./dist ./types", + "example": "npx ts-node src/index.ts -e openapi -m https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/cognitiveservices/OpenAI.Inference/main.tsp ./temp", + "test": "mocha" + }, + "author": "Microsoft Corporation", + "license": "MIT", + "type": "module", + "engines": { + "node": ">=18.0.0" + }, + "bin": { + "tsp-client": "cmd/tsp-client.js" + }, + "files": [ + "dist", + "cmd/tsp-client.js" + ], + "devDependencies": { + "@types/chai": "^4.3.5", + "@types/mocha": "^10.0.1", + "@types/node": "^20.4.8", + "@types/prompt-sync": "^4.2.1", + "chai": "^4.3.7", + "mocha": "^10.2.0", + "prettier": "^3.0.1", + "rimraf": "^5.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.1.6" + }, + "dependencies": { + "@azure/core-rest-pipeline": "^1.12.0", + "chalk": "^5.3.0", + "prompt-sync": "^4.2.0", + "yaml": "^2.3.1" + }, + "peerDependencies": { + "@typespec/compiler": ">=0.48.1 <1.0.0" + }, + "mocha": { + "extension": [ + "ts" + ], + "spec": "test/**/*.spec.ts", + "loader": "ts-node/esm" + } +} diff --git a/tools/tsp-client/src/fileTree.ts b/tools/tsp-client/src/fileTree.ts new file mode 100644 index 00000000000..cd10076a254 --- /dev/null +++ b/tools/tsp-client/src/fileTree.ts @@ -0,0 +1,79 @@ +export function createFileTree(url: string): FileTree { + const rootUrl = url; + const fileMap = new Map(); + + function longestCommonPrefix(a: string, b: string): string { + if (a === b) { + return a; + } + let lastCommonSlash = -1; + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (a[i] === b[i]) { + if (a[i] === "/") { + lastCommonSlash = i; + } + } else { + break; + } + } + if (lastCommonSlash === -1) { + throw new Error("no common prefix found"); + } + return a.slice(0, lastCommonSlash + 1); + } + + function findCommonRoot(): string { + let candidate = ""; + for (const fileUrl of fileMap.keys()) { + const lastSlashIndex = fileUrl.lastIndexOf("/"); + const dirUrl = fileUrl.slice(0, lastSlashIndex + 1); + if (!candidate) { + candidate = dirUrl; + } else { + candidate = longestCommonPrefix(candidate, dirUrl); + } + } + return candidate; + } + + return { + addFile(url: string, contents: string): void { + if (fileMap.has(url)) { + throw new Error(`file already parsed: ${url}`); + } + fileMap.set(url, contents); + }, + async createTree(): Promise { + const outputFiles = new Map(); + // calculate the highest common root + const root = findCommonRoot(); + let mainFilePath = ""; + for (const [url, contents] of fileMap.entries()) { + const relativePath = url.slice(root.length); + outputFiles.set(relativePath, contents); + if (url === rootUrl) { + mainFilePath = relativePath; + } + } + if (!mainFilePath) { + throw new RangeError( + `Main file ${rootUrl} not added to FileTree. Did you forget to add it?`, + ); + } + return { + mainFilePath, + files: outputFiles, + }; + }, + }; +} + +export interface FileTreeResult { + mainFilePath: string; + files: Map; +} + +export interface FileTree { + addFile(url: string, contents: string): void; + createTree(): Promise; +} diff --git a/tools/tsp-client/src/fs.ts b/tools/tsp-client/src/fs.ts new file mode 100644 index 00000000000..d874232762c --- /dev/null +++ b/tools/tsp-client/src/fs.ts @@ -0,0 +1,92 @@ +import { mkdir, rm, writeFile, stat, readFile, access } from "node:fs/promises"; +import { FileTreeResult } from "./fileTree.js"; +import * as path from "node:path"; +import { Logger } from "./log.js"; +import { parse as parseYaml } from "yaml"; + +export async function ensureDirectory(path: string) { + await mkdir(path, { recursive: true }); +} + +export async function removeDirectory(path: string) { + await rm(path, { recursive: true, force: true }); +} + +export async function createTempDirectory(outputDir: string): Promise { + const tempRoot = path.join(outputDir, "TempTypeSpecFiles"); + await mkdir(tempRoot, { recursive: true }); + Logger.debug(`Created temporary working directory ${tempRoot}`); + return tempRoot; +} + +export async function writeFileTree(rootDir: string, files: FileTreeResult["files"]) { + for (const [relativeFilePath, contents] of files) { + const filePath = path.join(rootDir, relativeFilePath); + await ensureDirectory(path.dirname(filePath)); + Logger.debug(`writing ${filePath}`); + await writeFile(filePath, contents); + } +} + +export async function tryReadTspLocation(rootDir: string): Promise { + try { + const yamlPath = path.resolve(rootDir, "tsp-location.yaml"); + const fileStat = await stat(yamlPath); + if (fileStat.isFile()) { + const fileContents = await readFile(yamlPath, "utf8"); + const locationYaml = parseYaml(fileContents); + const { directory, commit, repo } = locationYaml; + if (!directory || !commit || !repo) { + throw new Error("Invalid tsp-location.yaml"); + } + // make GitHub URL + return `https://raw.githubusercontent.com/${repo}/${commit}/${directory}/`; + } + } catch (e) { + Logger.error(`Error reading tsp-location.yaml: ${e}`); + } + return undefined; +} + +export async function readTspLocation(rootDir: string): Promise<[string, string, string, string[]]> { + try { + const yamlPath = path.resolve(rootDir, "tsp-location.yaml"); + const fileStat = await stat(yamlPath); + if (fileStat.isFile()) { + const fileContents = await readFile(yamlPath, "utf8"); + const locationYaml = parseYaml(fileContents); + let { directory, commit, repo, additionalDirectories } = locationYaml; + if (!directory || !commit || !repo) { + throw new Error("Invalid tsp-location.yaml"); + } + Logger.info(`Additional directories: ${additionalDirectories}`) + if (!additionalDirectories) { + additionalDirectories = []; + } + return [ directory, commit, repo, additionalDirectories ]; + } + throw new Error("Could not find tsp-location.yaml"); + } catch (e) { + Logger.error(`Error reading tsp-location.yaml: ${e}`); + throw e; + } +} + + +export async function getEmitterFromRepoConfig(emitterPath: string): Promise { + await access(emitterPath); + const data = await readFile(emitterPath, 'utf8'); + const obj = JSON.parse(data); + if (!obj || !obj.dependencies) { + throw new Error("Invalid emitter-package.json"); + } + const languages: string[] = ["@azure-tools/typespec-", "@typespec/openapi3"]; + for (const lang of languages) { + const emitter = Object.keys(obj.dependencies).find((dep: string) => dep.startsWith(lang)); + if (emitter) { + Logger.info(`Found emitter package ${emitter}`); + return emitter; + } + } + throw new Error("Could not find emitter package"); +} diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts new file mode 100644 index 00000000000..f29bf40d7ef --- /dev/null +++ b/tools/tsp-client/src/git.ts @@ -0,0 +1,82 @@ +import { execSync, spawn } from "child_process"; + +export function getRepoRoot(): string { + return execSync('git rev-parse --show-toplevel').toString().trim(); +} + +export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise { + return new Promise((resolve, reject) => { + const git = spawn("git", ["clone", "--no-checkout", "--filter=tree:0", repo, cloneDir], { + cwd: rootUrl, + stdio: "inherit", + }); + git.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`git clone failed exited with code ${code}`)); + } + }); + git.once("error", (err) => { + reject(new Error(`git clone failed with error: ${err}`)); + }); + }); + } + + + export async function sparseCheckout(cloneDir: string): Promise { + return new Promise((resolve, reject) => { + const git = spawn("git", ["sparse-checkout", "init"], { + cwd: cloneDir, + stdio: "inherit", + }); + git.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`git sparse-checkout failed exited with code ${code}`)); + } + }); + git.once("error", (err) => { + reject(new Error(`git sparse-checkout failed with error: ${err}`)); + }); + }); + } + + export async function addSpecFiles(cloneDir: string, subDir: string): Promise { + return new Promise((resolve, reject) => { + const git = spawn("git", ["sparse-checkout", "add", subDir], { + cwd: cloneDir, + stdio: "inherit", + }); + git.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`git sparse-checkout add failed exited with code ${code}`)); + } + }); + git.once("error", (err) => { + reject(new Error(`git sparse-checkout add failed with error: ${err}`)); + }); + }); + } + + export async function checkoutCommit(cloneDir: string, commit: string): Promise { + return new Promise((resolve, reject) => { + const git = spawn("git", ["checkout", commit], { + cwd: cloneDir, + stdio: "inherit", + }); + git.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`git checkout failed exited with code ${code}`)); + } + }); + git.once("error", (err) => { + reject(new Error(`git checkout failed with error: ${err}`)); + }); + }); + } diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts new file mode 100644 index 00000000000..c3ae6673ebd --- /dev/null +++ b/tools/tsp-client/src/index.ts @@ -0,0 +1,275 @@ +import * as path from "node:path"; + +import { installDependencies } from "./npm.js"; +import { createTempDirectory, removeDirectory,readTspLocation, getEmitterFromRepoConfig } from "./fs.js"; +import { Logger, printBanner, enableDebug, printVersion } from "./log.js"; +import { compileTsp, discoverMainFile, getEmitterOptions, resolveTspConfigUrl } from "./typespec.js"; +import { getOptions } from "./options.js"; +import { mkdir, writeFile, cp, readFile } from "node:fs/promises"; +import { addSpecFiles, checkoutCommit, cloneRepo, getRepoRoot, sparseCheckout } from "./git.js"; +import { fetch } from "./network.js"; +import { parse as parseYaml } from "yaml"; + + +async function sdkInit( + { + config, + outputDir, + emitter, + commit, + repo, + isUrl, + }: { + config: string; + outputDir: string; + emitter: string; + commit: string | undefined; + repo: string | undefined; + isUrl: boolean; + }): Promise { + if (isUrl) { + // URL scenario + const resolvedConfigUrl = resolveTspConfigUrl(config); + Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`) + const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl); + const configYaml = parseYaml(tspConfig); + if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]){ + const serviceDir = configYaml["parameters"]["service-dir"]["default"]; + if (serviceDir === undefined) { + Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) + } + Logger.debug(`Service directory: ${serviceDir}`) + const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; + const packageDir: string | undefined = configYaml?.options?.[emitter]?.["package-dir"]; + if (!packageDir) { + Logger.error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); + } + const newPackageDir = path.join(outputDir, serviceDir, packageDir!) + await mkdir(newPackageDir, { recursive: true }); + await writeFile( + path.join(newPackageDir, "tsp-location.yaml"), + `directory: ${resolvedConfigUrl.path}\ncommit: ${resolvedConfigUrl.commit}\nrepo: ${resolvedConfigUrl.repo}\nadditionalDirectories: ${additionalDirs}`); + return newPackageDir; + } else { + Logger.error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.") + } + } else { + // Local directory scenario + let configFile = path.join(config, "tspconfig.yaml") + const data = await readFile(configFile, "utf8"); + const configYaml = parseYaml(data); + if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]) { + const serviceDir = configYaml["parameters"]["service-dir"]["default"]; + var additionalDirs: string[] = []; + if (configYaml["parameters"]["dependencies"] && configYaml["parameters"]["dependencies"]["additionalDirectories"]) { + additionalDirs = configYaml["parameters"]["dependencies"]["additionalDirectories"]; + } + Logger.info(`Additional directories: ${additionalDirs}`) + let packageDir: string | undefined = undefined; + if (configYaml["options"][emitter] && configYaml["options"][emitter]["package-dir"]) { + packageDir = configYaml["options"][emitter]["package-dir"]; + } + if (packageDir === undefined) { + throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); + } + const newPackageDir = path.join(outputDir, serviceDir, packageDir) + await mkdir(newPackageDir, { recursive: true }); + configFile = configFile.replaceAll("\\", "/"); + const matchRes = configFile.match('.*/(?specification/.*)/tspconfig.yaml$') + var directory = ""; + if (matchRes) { + if (matchRes.groups) { + directory = matchRes.groups!["path"]!; + } + } + writeFile(path.join(newPackageDir, "tsp-location.yaml"), + `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirs}`); + return newPackageDir; + } + throw new Error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.") + } + throw new Error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."); +} + +async function syncTspFiles(outputDir: string, localSpecRepo?: string) { + const tempRoot = await createTempDirectory(outputDir); + + const repoRoot = getRepoRoot(); + Logger.debug(`Repo root is ${repoRoot}`); + if (repoRoot === undefined) { + throw new Error("Could not find repo root"); + } + const [ directory, commit, repo, additionalDirectories ] = await readTspLocation(outputDir); + const dirSplit = directory.split("/"); + let projectName = dirSplit[dirSplit.length - 1]; + Logger.debug(`Using project name: ${projectName}`) + if (projectName === undefined) { + projectName = "src"; + } + const srcDir = path.join(tempRoot, projectName); + await mkdir(srcDir, { recursive: true }); + const cloneDir = path.join(repoRoot, "..", "sparse-spec"); + await mkdir(cloneDir, { recursive: true }); + Logger.debug(`Created temporary sparse-checkout directory ${cloneDir}`); + + if (localSpecRepo) { + Logger.debug(`Using local spec directory: ${localSpecRepo}`); + function filter(src: string, dest: string): boolean { + if (src.includes("node_modules") || dest.includes("node_modules")) { + return false; + } + return true; + } + const cpDir = path.join(cloneDir, directory); + await cp(localSpecRepo, cpDir, { recursive: true, filter: filter }); + // TODO: additional directories not yet supported + // const localSpecRepoRoot = await getRepoRoot(localSpecRepo); + // if (localSpecRepoRoot === undefined) { + // throw new Error("Could not find local spec repo root, please make sure the path is correct"); + // } + // for (const dir of additionalDirectories) { + // await cp(path.join(localSpecRepoRoot, dir), cpDir, { recursive: true, filter: filter }); + // } + } else { + Logger.debug(`Cloning repo to ${cloneDir}`); + await cloneRepo(tempRoot, cloneDir, `https://github.com/${repo}.git`); + await sparseCheckout(cloneDir); + await addSpecFiles(cloneDir, directory) + Logger.info(`Processing additional directories: ${additionalDirectories}`) + for (const dir of additionalDirectories) { + await addSpecFiles(cloneDir, dir); + } + await checkoutCommit(cloneDir, commit); + } + + await cp(path.join(cloneDir, directory), srcDir, { recursive: true }); + const emitterPath = path.join(repoRoot, "eng", "emitter-package.json"); + await cp(emitterPath, path.join(srcDir, "package.json"), { recursive: true }); + // FIXME: remove conditional once full support for local spec repo is added + if (localSpecRepo) { + Logger.info("local spec repo does not yet support additional directories"); + } else { + for (const dir of additionalDirectories) { + const dirSplit = dir.split("/"); + let projectName = dirSplit[dirSplit.length - 1]; + if (projectName === undefined) { + projectName = "src"; + } + const dirName = path.join(tempRoot, projectName); + await cp(path.join(cloneDir, dir), dirName, { recursive: true }); + } + } + Logger.debug(`Removing sparse-checkout directory ${cloneDir}`); + await removeDirectory(cloneDir); +} + + +async function generate({ + rootUrl, + noCleanup, + additionalEmitterOptions, +}: { + rootUrl: string; + noCleanup: boolean; + additionalEmitterOptions?: string; +}) { + const tempRoot = path.join(rootUrl, "TempTypeSpecFiles"); + const tspLocation = await readTspLocation(rootUrl); + const dirSplit = tspLocation[0].split("/"); + let projectName = dirSplit[dirSplit.length - 1]; + if (projectName === undefined) { + throw new Error("cannot find project name"); + } + const srcDir = path.join(tempRoot, projectName); + const emitter = await getEmitterFromRepoConfig(path.join(getRepoRoot(), "eng", "emitter-package.json")); + if (!emitter) { + throw new Error("emitter is undefined"); + } + const mainFilePath = await discoverMainFile(srcDir); + const resolvedMainFilePath = path.join(srcDir, mainFilePath); + Logger.info(`Compiling tsp using ${emitter}...`); + const emitterOpts = await getEmitterOptions(rootUrl, srcDir, emitter, noCleanup, additionalEmitterOptions); + + Logger.info("Installing dependencies from npm..."); + await installDependencies(srcDir); + + await compileTsp({ emitterPackage: emitter, outputPath: rootUrl, resolvedMainFilePath, options: emitterOpts }); + + if (noCleanup) { + Logger.debug(`Skipping cleanup of temp directory: ${tempRoot}`); + } else { + Logger.debug("Cleaning up temp directory"); + await removeDirectory(tempRoot); + } +} + +async function syncAndGenerate({ + outputDir, + noCleanup, +}: { + outputDir: string; + noCleanup: boolean; +}) { + await syncTspFiles(outputDir); + await generate({ rootUrl: outputDir, noCleanup}); +} + +async function main() { + const options = await getOptions(); + if (options.debug) { + enableDebug(); + } + printBanner(); + await printVersion(); + + let rootUrl = path.resolve("."); + if (options.outputDir) { + rootUrl = path.resolve(options.outputDir); + } + + switch (options.command) { + case "init": + const emitter = await getEmitterFromRepoConfig(path.join(getRepoRoot(), "eng", "emitter-package.json")); + if (!emitter) { + throw new Error("Couldn't find emitter-package.json in the repo"); + } + const outputDir = await sdkInit({config: options.tspConfig!, outputDir: rootUrl, emitter, commit: options.commit, repo: options.repo, isUrl: options.isUrl}); + Logger.info(`SDK initialized in ${outputDir}`); + if (!options.skipSyncAndGenerate) { + await syncAndGenerate({outputDir, noCleanup: options.noCleanup}) + } + break; + case "sync": + syncTspFiles(rootUrl, options.localSpecRepo); + break; + case "generate": + generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); + break; + case "update": + if (options.repo && !options.commit) { + throw new Error("Commit SHA is required when specifying `--repo`, please specify a commit using `--commit`"); + } + if (options.commit) { + let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl); + commit = options.commit ?? commit; + repo = options.repo ?? repo; + await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`); + } + if (options.tspConfig) { + let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl); + let tspConfig = resolveTspConfigUrl(options.tspConfig); + commit = tspConfig.commit ?? commit; + repo = tspConfig.repo ?? repo; + await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`); + } + syncAndGenerate({outputDir: rootUrl, noCleanup: options.noCleanup}); + break; + default: + Logger.error(`Unknown command: ${options.command}`); + } +} + +main().catch((err) => { + Logger.error(err); + process.exit(1); +}); diff --git a/tools/tsp-client/src/languageSettings.ts b/tools/tsp-client/src/languageSettings.ts new file mode 100644 index 00000000000..1671d8db127 --- /dev/null +++ b/tools/tsp-client/src/languageSettings.ts @@ -0,0 +1,61 @@ +import { join } from "node:path"; + +interface LanguageEmitterSettings { + emitterName: string; + emitterOptions: Record; +} + +export const knownLanguages = ["csharp", "java", "javascript", "python", "openapi"] as const; +export type KnownLanguage = (typeof knownLanguages)[number]; +export const languageAliases: Record = { + cs: "csharp", + js: "javascript", + ts: "javascript", + typescript: "javascript", + py: "python", +}; + +export function getEmitterPackage(language: string): string { + if (language in languageEmitterSettings) { + return languageEmitterSettings[language as KnownLanguage].emitterName; + } + throw new Error(`Unknown language ${language}`); +} + +export function getEmitterOutputPath(language: string, projectDirectory: string): string { + if (language === "csharp") { + return join(projectDirectory, "src"); + } + return projectDirectory; +} + +const languageEmitterSettings: Record = { + csharp: { + emitterName: "@azure-tools/typespec-csharp", + emitterOptions: { + "emitter-output-dir": "$projectDirectory/src", + }, + }, + java: { + emitterName: "@azure-tools/typespec-java", + emitterOptions: { + "emitter-output-dir": "$projectDirectory", + }, + }, + javascript: { + emitterName: "@azure-tools/typespec-ts", + emitterOptions: { + "emitter-output-dir": "$projectDirectory", + }, + }, + python: { + emitterName: "@azure-tools/typespec-python", + emitterOptions: { + "emitter-output-dir": "$projectDirectory", + }, + }, + openapi: { + emitterName: "@typespec/openapi3", + emitterOptions: {}, + }, +}; diff --git a/tools/tsp-client/src/log.ts b/tools/tsp-client/src/log.ts new file mode 100644 index 00000000000..569364c8438 --- /dev/null +++ b/tools/tsp-client/src/log.ts @@ -0,0 +1,86 @@ +import chalk from "chalk"; +import { getPackageVersion } from "./npm.js"; + +const logSink = { + log: console.log, + info: console.info, + warn: console.warn, + error: console.error, + debug: (..._args: unknown[]) => { + /* no-op */ + }, +}; + +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; + debug(message: string): void; + success(message: string): void; + (message: string): void; +} + +function createLogger(): Logger { + const direct = (...values: unknown[]) => logSink.log(chalk.reset(...values)); + direct.info = (...values: unknown[]) => logSink.info(chalk.blueBright(...values)); + direct.warn = (...values: unknown[]) => logSink.warn(chalk.yellow(...values)); + direct.error = (...values: unknown[]) => logSink.error(chalk.red(...values)); + direct.debug = (...values: unknown[]) => logSink.debug(chalk.magenta(...values)); + direct.success = (...values: unknown[]) => logSink.info(chalk.green(...values)); + + return direct; +} + +export function enableDebug(): void { + logSink.debug = console.debug; +} + +export const Logger = createLogger(); + +const bannerText = ` +888 888 d8b 888 +888 888 Y8P 888 +888 888 888 +888888 .d8888b 88888b. .d8888b 888 888 .d88b. 88888b. 888888 +888 88K 888 "88b d88P" 888 888 d8P Y8b 888 "88b 888 +888 "Y8888b. 888 888 888888 888 888 888 88888888 888 888 888 +Y88b. X88 888 d88P Y88b. 888 888 Y8b. 888 888 Y88b. + "Y888 88888P' 88888P" "Y8888P 888 888 "Y8888 888 888 "Y888 + 888 + 888 + 888 +`; +export function printBanner() { + Logger.info(bannerText); +} + +const usageText = ` +Usage: tsp-client [options] + +Generate from a tsp file using --mainFile or use tsp-location.yaml inside +the outputDir. + +Positionals: + init Initialize the SDK project folder from a tspconfig.yaml [string] + sync Sync tsp files using tsp-location.yaml [string] + generate Generate from a tsp project [string] + update Sync and generate from a tsp project [string] + +Options: + -d, --debug Enable debug logging [boolean] + -e, --emitter Which language emitter to use [required] [string] + [choices: "csharp", "java", "javascript", "python", "openapi"] + -m, --mainFile The url of the main tsp file to generate from [string] + --noCleanup Don't clean up the temp directory after generation [boolean] + -h, --help Show help [boolean] + -v, --version Show version number [boolean] + -o, --outputDir The output directory for the emitter +`; +export function printUsage() { + Logger(usageText); +} + +export async function printVersion() { + const version = await getPackageVersion(); + Logger(`tsp-client version: ${version}`); +} diff --git a/tools/tsp-client/src/network.ts b/tools/tsp-client/src/network.ts new file mode 100644 index 00000000000..dabac0e22a3 --- /dev/null +++ b/tools/tsp-client/src/network.ts @@ -0,0 +1,92 @@ +import { createDefaultHttpClient, createPipelineRequest } from "@azure/core-rest-pipeline"; +import { createFileTree } from "./fileTree.js"; +import { resolveImports } from "./typespec.js"; +import { Logger } from "./log.js"; + +const httpClient = createDefaultHttpClient(); + +export async function fetch(url: string, method: "GET" | "HEAD" = "GET"): Promise { + const result = await httpClient.sendRequest(createPipelineRequest({ url, method })); + if (result.status !== 200) { + throw new Error(`failed to fetch ${url}: ${result.status}`); + } + return String(result.bodyAsText); +} + +export function isValidUrl(url: string) { + try { + new URL(url); + return true; + } catch { + return false; + } +} + +export function doesFileExist(url: string): Promise { + return fetch(url, "HEAD") + .then(() => true) + .catch(() => false); +} + +export function rewriteGitHubUrl(url: string): string { + if (url.includes("github.com")) { + const result = url.replace("github.com", "raw.githubusercontent.com"); + Logger.debug(`rewriting github url to direct link: ${result}`); + return result; + } + return url; +} + +export async function downloadTsp(rootUrl: string) { + const seenFiles = new Set(); + const moduleImports = new Set(); + // fetch the root file + const filesToProcess = [rootUrl]; + seenFiles.add(rootUrl); + const fileTree = createFileTree(rootUrl); + + while (filesToProcess.length > 0) { + const url = filesToProcess.shift()!; + const sourceFile = await fetch(url); + fileTree.addFile(url, sourceFile); + // process imports, fetching any relatively referenced files + const imports = await resolveImports(sourceFile); + for (const fileImport of imports) { + // Check if the module name is referencing a path(./foo, /foo, file:/foo) + if (/^(?:\.\.?(?:\/|$)|\/|([A-Za-z]:)?[/\\])/.test(fileImport)) { + if (fileImport.startsWith("file:")) { + throw new Error(`file protocol imports are not supported: ${fileImport}`); + } + let resolvedImport: string; + if (fileImport.startsWith("http:")) { + throw new Error(`absolute url imports are not supported: ${fileImport}`); + } else { + resolvedImport = new URL(fileImport, url).toString(); + } + if (!seenFiles.has(resolvedImport)) { + Logger.debug(`discovered import ${resolvedImport}`); + filesToProcess.push(resolvedImport); + seenFiles.add(resolvedImport); + } + } else { + Logger.debug(`discovered module import ${fileImport}`); + moduleImports.add(fileImport); + } + } + } + + // look for a tspconfig.yaml next to the root + try { + const tspConfigUrl = new URL("tspconfig.yaml", rootUrl).toString(); + const tspConfig = await fetch(tspConfigUrl); + Logger.debug("found tspconfig.yaml"); + fileTree.addFile(tspConfigUrl, tspConfig); + } catch (e) { + Logger.debug("no tspconfig.yaml found"); + } + + return { + moduleImports, + fileTree, + }; +} diff --git a/tools/tsp-client/src/npm.ts b/tools/tsp-client/src/npm.ts new file mode 100644 index 00000000000..bd3ca6ca6c1 --- /dev/null +++ b/tools/tsp-client/src/npm.ts @@ -0,0 +1,51 @@ +import * as path from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +export async function createPackageJson(rootPath: string, deps: Set): Promise { + const dependencies: Record = {}; + + for (const dep of deps) { + dependencies[dep] = "latest"; + } + + const packageJson = JSON.stringify({ + dependencies, + }); + + const filePath = path.join(rootPath, "package.json"); + await writeFile(filePath, packageJson); +} + +export function installDependencies(workingDir: string): Promise { + return new Promise((resolve, reject) => { + const npm = spawn("npm", ["install", "--no-lock-file"], { + cwd: workingDir, + stdio: "inherit", + shell: true, + }); + npm.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`npm failed exited with code ${code}`)); + } + }); + npm.once("error", (err) => { + reject(new Error(`npm install failed with error: ${err}`)); + }); + }); +} + +let packageVersion: string; +export async function getPackageVersion(): Promise { + if (!packageVersion) { + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const packageJson = JSON.parse( + await readFile(path.join(__dirname, "..", "package.json"), "utf-8"), + ); + packageVersion = packageJson.version ?? "unknown"; + } + return packageVersion; +} diff --git a/tools/tsp-client/src/options.ts b/tools/tsp-client/src/options.ts new file mode 100644 index 00000000000..bbefda0a33a --- /dev/null +++ b/tools/tsp-client/src/options.ts @@ -0,0 +1,168 @@ +import { parseArgs } from "node:util"; +import { Logger, printUsage, printVersion } from "./log.js"; +import * as path from "node:path"; +import { knownLanguages, languageAliases } from "./languageSettings.js"; +import { doesFileExist } from "./network.js"; +import PromptSync from "prompt-sync"; + +export interface Options { + debug: boolean; + command: string; + tspConfig?: string; + emitter?: string; + mainFile?: string; + outputDir: string; + noCleanup: boolean; + skipSyncAndGenerate: boolean; + commit?: string; + repo?: string; + isUrl: boolean; + localSpecRepo?: string; + emitterOptions?: string; +} + +export async function getOptions(): Promise { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + help: { + type: "boolean", + short: "h", + }, + version: { + type: "boolean", + short: "v", + }, + debug: { + type: "boolean", + short: "d", + }, + emitter: { + type: "string", + short: "e", + }, + mainFile: { + type: "string", + short: "m", + }, + outputDir: { + type: "string", + short: "o", + }, + tspConfig: { + type: "string", + short: "c", + }, + commit: { + type: "string", + short: "C", + }, + repo: { + type: "string", + short: "R", + }, + ["emitter-options"]: { + type: "string", + }, + ["local-spec-repo"]: { + type: "string", + }, + ["save-inputs"]: { + type: "boolean", + }, + ["skip-sync-and-generate"]: { + type: "boolean", + } + }, + }); + if (values.help) { + printUsage(); + process.exit(0); + } + + if (values.version) { + await printVersion(); + process.exit(0); + } + + if (values.emitter) { + let emitter = values.emitter.toLowerCase(); + if (emitter in languageAliases) { + emitter = languageAliases[emitter]!; + } + if (!(knownLanguages as readonly string[]).includes(emitter)) { + Logger.error(`Unknown language ${values.emitter}`); + Logger.error(`Valid languages are: ${knownLanguages.join(", ")}`); + printUsage(); + process.exit(1); + } + } + + if (positionals.length === 0) { + Logger.error("Command is required"); + printUsage(); + process.exit(1); + } + + if (positionals[0] !== "sync" && positionals[0] !== "generate" && positionals[0] !== "update" && positionals[0] !== "init") { + Logger.error(`Unknown command ${positionals[0]}`); + printUsage(); + process.exit(1); + } + + let isUrl = false; + if (positionals[0] === "init") { + if (!values.tspConfig) { + Logger.error("tspConfig is required"); + printUsage(); + process.exit(1); + } + if (await doesFileExist(values.tspConfig)) { + isUrl = true; + } + if (!isUrl) { + if (values.commit === undefined || values.repo === undefined) { + Logger.error("The commit and repo options are required when tspConfig is a local directory"); + printUsage(); + process.exit(1); + } + } + } + // By default, assume that the command is run from the output directory + let outputDir = "."; + if (values.outputDir) { + outputDir = values.outputDir; + } + outputDir = path.resolve(path.normalize(outputDir)); + + // Ask user is this is the correct output directory + const prompt = PromptSync(); + let useOutputDir = prompt("Use output directory '" + outputDir + "'? (y/n) ", "y"); + + if (useOutputDir.toLowerCase() === "n") { + const newOutputDir = prompt("Enter output directory: "); + if (!newOutputDir) { + Logger.error("Output directory is required"); + printUsage(); + process.exit(1); + } + outputDir = path.resolve(path.normalize(newOutputDir)); + } + Logger.info("Using output directory '" + outputDir + "'"); + + return { + debug: values.debug ?? false, + command: positionals[0], + tspConfig: values.tspConfig, + emitter: values.emitter, + mainFile: values.mainFile, + noCleanup: values["save-inputs"] ?? false, + skipSyncAndGenerate: values["skip-sync-and-generate"] ?? false, + outputDir: outputDir, + commit: values.commit, + repo: values.repo, + isUrl: isUrl, + localSpecRepo: values["local-spec-repo"], + emitterOptions: values["emitter-options"], + }; +} diff --git a/tools/tsp-client/src/typespec.ts b/tools/tsp-client/src/typespec.ts new file mode 100644 index 00000000000..3c00fc69d0d --- /dev/null +++ b/tools/tsp-client/src/typespec.ts @@ -0,0 +1,150 @@ +import { NodeHost, compile, getSourceLocation } from "@typespec/compiler"; +import { parse, isImportStatement } from "@typespec/compiler"; +import { Logger } from "./log.js"; +import { readFile, readdir } from "fs/promises"; +import * as path from "node:path"; +import { parse as parseYaml } from "yaml"; + + +export async function getEmitterOptions(rootUrl: string, tempRoot: string, emitter: string, saveInputs: boolean, additionalOptions?: string): Promise> { + // TODO: Add a way to specify emitter options like Language-Settings.ps1, could be a languageSettings.ts file + let emitterOptions: Record> = {}; + emitterOptions[emitter] = {}; + if (additionalOptions) { + emitterOptions = resolveCliOptions(additionalOptions.split(",")); + } else { + const configData = await readFile(path.join(tempRoot, "tspconfig.yaml"), "utf8"); + const configYaml = parseYaml(configData); + emitterOptions[emitter] = configYaml?.options?.[emitter]; + // TODO: This accounts for a very specific and common configuration in the tspconfig.yaml files, + // we should consider making this more generic. + Object.keys(emitterOptions[emitter]!).forEach((key) => { + if (emitterOptions![emitter]![key] === "{package-dir}") { + emitterOptions![emitter]![key] = emitterOptions![emitter]!["package-dir"]; + } + }); + } + if (!emitterOptions?.[emitter]?.["emitter-output-dir"]) { + emitterOptions[emitter]!["emitter-output-dir"] = rootUrl; + } + if (saveInputs) { + if (emitterOptions[emitter] === undefined) { + emitterOptions[emitter] = {}; + } + emitterOptions[emitter]!["save-inputs"] = true; + } + Logger.debug(`Using emitter options: ${JSON.stringify(emitterOptions)}`); + return emitterOptions; +} + +export function resolveCliOptions(opts: string[]): Record> { + const options: Record> = {}; + for (const option of opts ?? []) { + const optionParts = option.split("="); + if (optionParts.length !== 2) { + throw new Error( + `The --option parameter value "${option}" must be in the format: .some-options=value` + ); + } + let optionKeyParts = optionParts[0]!.split("."); + if (optionKeyParts.length > 2) { + // support emitter/path/file.js.option=xyz + optionKeyParts = [ + optionKeyParts.slice(0, -1).join("."), + optionKeyParts[optionKeyParts.length - 1]!, + ]; + } + let emitterName = optionKeyParts[0]; + emitterName = emitterName?.replace(".", "/") + const key = optionKeyParts[1]; + if (!(emitterName! in options)) { + options[emitterName!] = {}; + } + options[emitterName!]![key!] = optionParts[1]!; + } + return options; +} + + +export function resolveTspConfigUrl(configUrl: string): { + resolvedUrl: string; + commit: string; + repo: string; + path: string; +} { + let resolvedConfigUrl = configUrl; + + const res = configUrl.match('^https://(?github|raw.githubusercontent).com/(?[^/]*/azure-rest-api-specs(-pr)?)/(tree/|blob/)?(?[0-9a-f]{40})/(?.*)/tspconfig.yaml$') + if (res && res.groups) { + if (res.groups["urlRoot"]! === "github") { + resolvedConfigUrl = configUrl.replace("github.com", "raw.githubusercontent.com"); + resolvedConfigUrl = resolvedConfigUrl.replace("/blob/", "/"); + } + return { + resolvedUrl: resolvedConfigUrl, + commit: res.groups!["commit"]!, + repo: res.groups!["repo"]!, + path: res.groups!["path"]!, + } + } else { + throw new Error(`Invalid tspconfig.yaml url: ${configUrl}`); + } +} + + +export async function discoverMainFile(srcDir: string): Promise { + Logger.debug(`Discovering entry file in ${srcDir}`) + let entryTsp = ""; + const files = await readdir(srcDir, {recursive: true }); + for (const file of files) { + if (file.includes("client.tsp") || file.includes("main.tsp")) { + entryTsp = file; + Logger.debug(`Found entry file: ${entryTsp}`); + return entryTsp; + } + }; + throw new Error(`No main.tsp or client.tsp found`); +} + +export async function resolveImports(file: string): Promise { + const imports: string[] = []; + const node = await parse(file); + for (const statement of node.statements) { + if (isImportStatement(statement)) { + imports.push(statement.path.value); + } + } + return imports; +} + +export async function compileTsp({ + emitterPackage, + outputPath, + resolvedMainFilePath, + options, +}: { + emitterPackage: string; + outputPath: string; + resolvedMainFilePath: string; + options: Record>; +}) { + Logger.debug(`Using emitter output dir: ${outputPath}`); + // compile the local copy of the root file + const program = await compile(NodeHost, resolvedMainFilePath, { + outputDir: outputPath, + emit: [emitterPackage], + options: options, + }); + + if (program.diagnostics.length > 0) { + for (const diagnostic of program.diagnostics) { + const location = getSourceLocation(diagnostic.target); + const source = location ? location.file.path : "unknown"; + console.error( + `${diagnostic.severity}: ${diagnostic.code} - ${diagnostic.message} @ ${source}`, + ); + } + } else { + Logger.success("generation complete"); + } +} diff --git a/tools/tsp-client/test/fileTree.spec.ts b/tools/tsp-client/test/fileTree.spec.ts new file mode 100644 index 00000000000..7c4f8bab527 --- /dev/null +++ b/tools/tsp-client/test/fileTree.spec.ts @@ -0,0 +1,21 @@ +import { assert } from "chai"; +import { createFileTree } from "../src/fileTree.js"; + +describe("FileTree", function () { + it("handles basic tree", async function () { + const tree = createFileTree("http://example.com/foo/bar/baz.tsp"); + tree.addFile("http://example.com/foo/bar/baz.tsp", "text"); + tree.addFile("http://example.com/foo/buzz/foo.tsp", "text"); + tree.addFile("http://example.com/foo/bar/qux.tsp", "text"); + const result = await tree.createTree(); + assert.strictEqual(result.mainFilePath, "bar/baz.tsp"); + assert.strictEqual(result.files.size, 3); + const resultPaths = new Set(result.files.keys()); + assert.isTrue(resultPaths.has("bar/baz.tsp")); + assert.isTrue(resultPaths.has("bar/qux.tsp")); + assert.isTrue(resultPaths.has("buzz/foo.tsp")); + for (const contents of result.files.values()) { + assert.strictEqual(contents, "text"); + } + }); +}); diff --git a/tools/tsp-client/test/network.spec.ts b/tools/tsp-client/test/network.spec.ts new file mode 100644 index 00000000000..9f6f7f69cb1 --- /dev/null +++ b/tools/tsp-client/test/network.spec.ts @@ -0,0 +1,13 @@ +import { assert } from "chai"; +import { rewriteGitHubUrl } from "../src/network.js"; + +describe("Network", function () { + it("rewriteGitHubUrl", function () { + const initial = + "https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/OpenAI.Inference/main.tsp"; + const expected = + "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/cognitiveservices/OpenAI.Inference/main.tsp"; + const actual = rewriteGitHubUrl(initial); + assert.strictEqual(actual, expected); + }); +}); diff --git a/tools/tsp-client/tsconfig.json b/tools/tsp-client/tsconfig.json new file mode 100644 index 00000000000..7fd53993e28 --- /dev/null +++ b/tools/tsp-client/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "outDir": "dist", + "target": "ES2022", + "module": "Node16", + "moduleResolution": "node16", + "strict": true, + "noImplicitAny": true, + "noUnusedLocals": true, + "noFallthroughCasesInSwitch": true, + "noUnusedParameters": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noImplicitThis": true, + "noImplicitReturns": true, + "noPropertyAccessFromIndexSignature": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"], + "ts-node": { + "esm": true + } +} From dc70214097b414336784b909272966a9973a207d Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 10 Oct 2023 09:55:33 -0700 Subject: [PATCH 54/93] Remove CXPAttention rules constant (#7092) --- .../Constants/RulesConstants.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs index 13d23c37488..90b05aa188c 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/RulesConstants.cs @@ -16,7 +16,6 @@ public class RulesConstants public const string InitialIssueTriage = "InitialIssueTriage"; public const string ManualIssueTriage = "ManualIssueTriage"; public const string ServiceAttention = "ServiceAttention"; - public const string CXPAttention = "CXPAttention"; public const string ManualTriageAfterExternalAssignment = "ManualTriageAfterExternalAssignment"; public const string RequireAttentionForNonMilestone = "RequireAttentionForNonMilestone"; public const string AuthorFeedbackNeeded = "AuthorFeedbackNeeded"; From fad5a2e3e770c057af37ecd8e6a8eca2cbd2ab0e Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Tue, 10 Oct 2023 13:36:18 -0700 Subject: [PATCH 55/93] Allow overrides when creating emitter package json (#7094) --- .../stages/archetype-autorest-preview.yml | 27 +++----- .../autorest/New-EmitterPackageJson.ps1 | 61 +++++++++++++------ .../autorest/New-EmitterPackageLock.ps1 | 4 +- 3 files changed, 53 insertions(+), 39 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-autorest-preview.yml b/eng/pipelines/templates/stages/archetype-autorest-preview.yml index 862fca2b435..6a1388cd268 100644 --- a/eng/pipelines/templates/stages/archetype-autorest-preview.yml +++ b/eng/pipelines/templates/stages/archetype-autorest-preview.yml @@ -1,6 +1,6 @@ parameters: # Whether to build alpha versions of the packages. This is passed as a flag to the build script. -- name: BuildAlphaVersion +- name: BuildPrereleaseVersion type: boolean # Whether to use the `next` version of TypeSpec. This is passed as a flag to the init script. @@ -94,7 +94,8 @@ stages: - script: > npm run ci-build -- - --buildAlphaVersion ${{ parameters.BuildAlphaVersion }} + --publishTarget ${{ parameters.PublishTarget }} + --buildPrereleaseVersion ${{ parameters.BuildPrereleaseVersion }} --buildNumber $(Build.BuildNumber) --output $(Build.ArtifactStagingDirectory) displayName: 'Run build script' @@ -109,6 +110,7 @@ stages: filePath: $(toolsRepositoryPath)/eng/scripts/autorest/New-EmitterPackageJson.ps1 arguments: > -PackageJsonPath '${{ parameters.EmitterPackageJsonPath }}' + -OverridesPath $(Build.ArtifactStagingDirectory)/overrides.json -OutputDirectory '$(Build.ArtifactStagingDirectory)' workingDirectory: $(autorestRepositoryPath) @@ -147,15 +149,11 @@ stages: - stage: Publish dependsOn: Build variables: - autorestRepositoryPath: $(Build.SourcesDirectory)/autorest - toolsRepositoryPath: $(Build.SourcesDirectory)/azure-sdk-tools - sdkRepositoryPath: $(Build.SourcesDirectory)/azure-sdk + toolsRepositoryPath: $(Build.SourcesDirectory) buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts jobs: - job: Publish steps: - - checkout: self - path: s/autorest - checkout: azure-sdk-tools - download: current @@ -189,9 +187,8 @@ stages: inputs: command: 'push' packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} - # Nuget packages are always published to the same internal feed. PublishTarget doesn't affect this. + # Nuget packages are always published to the same internal feed https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net nuGetFeedType: 'internal' - # Publish to https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' - ${{ if ne(parameters.EmitterPackageJsonPath, 'not-specified') }}: @@ -200,15 +197,9 @@ stages: inputs: pwsh: true filePath: $(toolsRepositoryPath)/eng/scripts/autorest/New-EmitterPackageLock.ps1 - ${{ if eq(parameters.PublishTarget, 'internal') }}: - arguments: > - -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -NpmrcPath "$(buildArtifactsPath)/packages/.npmrc" - ${{ elseif eq(parameters.PublishTarget, 'public') }}: - arguments: > - -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" - -OutputDirectory "$(Build.ArtifactStagingDirectory)" + arguments: > + -EmitterPackageJsonPath "$(buildArtifactsPath)/emitter-package.json" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" - publish: $(Build.ArtifactStagingDirectory) artifact: publish_artifacts diff --git a/eng/scripts/autorest/New-EmitterPackageJson.ps1 b/eng/scripts/autorest/New-EmitterPackageJson.ps1 index a45206282b8..01e883d068c 100644 --- a/eng/scripts/autorest/New-EmitterPackageJson.ps1 +++ b/eng/scripts/autorest/New-EmitterPackageJson.ps1 @@ -1,8 +1,13 @@ [CmdletBinding()] param ( [parameter(Mandatory = $true)] + [ValidateScript({ Test-Path $_ })] [string]$PackageJsonPath, + [parameter(Mandatory = $false)] + [ValidateScript({ Test-Path $_ })] + [string]$OverridesPath, + [parameter(Mandatory = $true)] [string]$OutputDirectory, @@ -10,41 +15,59 @@ param ( [string]$PackageJsonFileName = "emitter-package.json" ) -$knownPackages = @( - "@azure-tools/typespec-azure-core" - "@azure-tools/typespec-client-generator-core" - "@typespec/compiler" - "@typespec/eslint-config-typespec" - "@typespec/http" - "@typespec/rest" - "@typespec/versioning" -) +$packageJson = Get-Content $PackageJsonPath | ConvertFrom-Json -AsHashtable + +# If we provide OverridesPath, use that to load a hashtable of version overrides +$overrides = @{} + +if ($OverridesPath) { + Write-Host "Using overrides from $OverridesPath`:`n" + $overrides = Get-Content $OverridesPath | ConvertFrom-Json -AsHashtable + Write-Host ($overrides | ConvertTo-Json) + Write-Host "" +} + -$packageJson = Get-Content $PackageJsonPath | ConvertFrom-Json +# If there's a peer dependency and a dev dependency for the same package, carry the +# dev dependency forward into emitter-package.json $devDependencies = @{} -foreach ($package in $knownPackages) { - $pinnedVersion = $packageJson.devDependencies.$package - if ($pinnedVersion) { +foreach ($package in $packageJson.peerDependencies.Keys) { + $pinnedVersion = $packageJson.devDependencies[$package] + if ($pinnedVersion -and -not $overrides[$package]) { + Write-Host "Pinning $package to $pinnedVersion" $devDependencies[$package] = $pinnedVersion } } $emitterPackageJson = [ordered]@{ - "main" = "dist/src/index.js" - "dependencies" = @{ - $packageJson.name = $packageJson.version - } + "main" = "dist/src/index.js" + "dependencies" = @{ + $packageJson.name = $overrides[$packageJson.name] ?? $packageJson.version + } } +# you shouldn't specify the same package in both dependencies and overrides +$overrides.Remove($packageJson.name) + +# Avoid adding an empty devDependencies section if($devDependencies.Keys.Count -gt 0) { - $emitterPackageJson["devDependencies"] = $devDependencies + $emitterPackageJson["devDependencies"] = $devDependencies +} + +# Avoid adding an empty overrides section +if($overrides.Keys.Count -gt 0) { + $emitterPackageJson["overrides"] = $overrides } New-Item $OutputDirectory -ItemType Directory -ErrorAction SilentlyContinue | Out-Null $OutputDirectory = Resolve-Path $OutputDirectory $dest = Join-Path $OutputDirectory $PackageJsonFileName +$destJson = $emitterPackageJson | ConvertTo-Json -Depth 100 + Write-Host "Generating $dest" -$emitterPackageJson | ConvertTo-Json -Depth 100 | Out-File $dest +$destJson | Out-File $dest + +Write-Host $destJson diff --git a/eng/scripts/autorest/New-EmitterPackageLock.ps1 b/eng/scripts/autorest/New-EmitterPackageLock.ps1 index e4ffb27edba..1872e104e4d 100644 --- a/eng/scripts/autorest/New-EmitterPackageLock.ps1 +++ b/eng/scripts/autorest/New-EmitterPackageLock.ps1 @@ -42,8 +42,8 @@ try { exit $LASTEXITCODE } - Write-Host '##[group]npm list --all --package-lock-only' - npm list --all --package-lock-only + Write-Host '##[group]npm list --all' + npm list --all Write-Host '##[endgroup]' $dest = Join-Path $OutputDirectory $LockFileName From 7d723b9d143b64ae9166ad52d4515bd32b9ef402 Mon Sep 17 00:00:00 2001 From: Libba Lawrence Date: Tue, 10 Oct 2023 14:26:17 -0700 Subject: [PATCH 56/93] [Pylint] Add bug fix for mgmt-core (#7095) * update to ignore mgmt core * update changelog --- .../azure-pylint-guidelines-checker/CHANGELOG.md | 1 + .../pylint_guidelines_checker.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md index d0cce88a725..5e20c9dbce8 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.2.0 (Unreleased) - Checker to enforce docstring keywords being keyword-only in method signature. +- Fixed a bug in `no-legacy-azure-core-http-response-import` that was throwing warnings for azure-mgmt-core. ## 0.1.0 (2023-08-02) diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py index 8dd0223f2ff..ad01fb5b91a 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py @@ -2288,12 +2288,13 @@ class NoLegacyAzureCoreHttpResponseImport(BaseChecker): } AZURE_CORE_NAME = "azure.core" + AZURE_MGMT_CORE = "azure.mgmt.core" AZURE_CORE_TRANSPORT_NAME = "azure.core.pipeline.transport" RESPONSE_CLASSES = ["HttpResponse", "AsyncHttpResponse"] def visit_importfrom(self, node): """Check that we aren't importing from azure.core.pipeline.transport import HttpResponse.""" - if node.root().name.startswith(self.AZURE_CORE_NAME): + if node.root().name.startswith(self.AZURE_CORE_NAME) or node.root().name.startswith(self.AZURE_MGMT_CORE): return if node.modname == self.AZURE_CORE_TRANSPORT_NAME: for name, _ in node.names: From 37a24747a883f3b63a71196a3fa3d0eb26b356fd Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 10 Oct 2023 16:10:49 -0700 Subject: [PATCH 57/93] Update github-event-processor version to 1.0.0-dev.20231010.2 (#7098) --- .github/workflows/event-processor.yml | 2 +- .github/workflows/scheduled-event-processor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/event-processor.yml b/.github/workflows/event-processor.yml index 56845c175cd..eef39a27b5b 100644 --- a/.github/workflows/event-processor.yml +++ b/.github/workflows/event-processor.yml @@ -55,7 +55,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230929.3 + --version 1.0.0-dev.20231010.2 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash diff --git a/.github/workflows/scheduled-event-processor.yml b/.github/workflows/scheduled-event-processor.yml index d8cc88325dd..d268467b071 100644 --- a/.github/workflows/scheduled-event-processor.yml +++ b/.github/workflows/scheduled-event-processor.yml @@ -34,7 +34,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20230929.3 + --version 1.0.0-dev.20231010.2 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash From 1e1ef7fd00059ecbfac351775e3eb8f26a8ebb62 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 11 Oct 2023 13:48:04 -0700 Subject: [PATCH 58/93] Use url version spec for autorest.* packages (#7067) --- eng/common/scripts/TypeSpec-Project-Generate.ps1 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/eng/common/scripts/TypeSpec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 index 05a0e0bdfd4..9f2c2804db0 100644 --- a/eng/common/scripts/TypeSpec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -53,13 +53,6 @@ function NpmInstallForProject([string]$workingDirectory) { Copy-Item -Path $emitterPackageLock -Destination "package-lock.json" -Force } - $useAlphaNpmRegistry = (Get-Content $replacementPackageJson -Raw).Contains("-alpha.") - - if($useAlphaNpmRegistry) { - Write-Host "Package.json contains '-alpha.' in the version, Creating .npmrc using public/azure-sdk-for-js-test-autorest feed." - "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest@local/npm/registry/ `n`nalways-auth=true" | Out-File '.npmrc' - } - if ($usingLockFile) { Invoke-LoggedCommand "npm ci" } From d53fa1ff323055794b61d993dc1acc8cef4bc696 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Wed, 11 Oct 2023 22:47:36 -0700 Subject: [PATCH 59/93] [Perf Automation] Improve .NET package version update logic (#7114) - Update Azure.Core.TestFramework.csproj to ensure consistent versions - If reference is missing, ignore instead of throwing - Required when updating multiple projects which may not contain all references --- .../Azure.Sdk.Tools.PerfAutomation/Net.cs | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs index 9ed0956ebd8..ba996916b9f 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs @@ -12,6 +12,9 @@ public class Net : LanguageBase { protected override Language Language => Language.Net; + private string CoreTestFrameworkProjectFile => + Path.Combine(WorkingDirectory, "sdk", "core", "Azure.Core.TestFramework", "src", "Azure.Core.TestFramework.csproj"); + // Azure.Core.TestFramework.TestEnvironment requires publishing under the "artifacts" folder to find the repository root. private string PublishDirectory => Path.Join(WorkingDirectory, "artifacts", "perf"); @@ -24,24 +27,41 @@ public class Net : LanguageBase { var projectFile = Path.Combine(WorkingDirectory, project); + await UpdatePackageVersions(projectFile, packageVersions); + + // All perf projects reference Azure.Core.TestFramework. Update it to ensure consistent versions. + await UpdatePackageVersions(CoreTestFrameworkProjectFile, packageVersions); + + Util.DeleteIfExists(PublishDirectory); + + var additionalBuildArguments = ""; + if (packageVersions.Values.Contains(Program.PackageVersionSource)) + { + // Force all transitive dependencies to use project references, to ensure all packages are built from source. + // The default is for transitive dependencies to use package references to the latest published version. + additionalBuildArguments += "-p:UseProjectReferenceToAzureClients=true"; + } + + // Disable source link, since it's not needed for perf runs, and also fails in sparse checkout repos + var processArguments = $"publish -c release -f net{languageVersion}.0 -o {PublishDirectory} -p:EnableSourceLink=false {additionalBuildArguments} {project}"; + + var result = await Util.RunAsync("dotnet", processArguments, workingDirectory: WorkingDirectory); + + return (result.StandardOutput, result.StandardError, null); + } + + private static async Task UpdatePackageVersions(string projectFile, IDictionary packageVersions) + { File.Copy(projectFile, projectFile + ".bak", overwrite: true); var projectContents = File.ReadAllText(projectFile); - var additionalBuildArguments = String.Empty; foreach (var v in packageVersions) { var packageName = v.Key; var packageVersion = v.Value; - if (packageVersion == Program.PackageVersionSource) - { - // Force all transitive dependencies to use project references, to ensure all packages are build from source. - // The default is for transitive dependencies to use package references to the latest published version. - additionalBuildArguments = "-p:UseProjectReferenceToAzureClients=true"; - } - else - { + if (packageVersion != Program.PackageVersionSource) { // TODO: Use XmlDocument instead of Regex // Existing reference might be to package or project: @@ -62,8 +82,7 @@ public class Net : LanguageBase } else { - throw new InvalidOperationException( - $"Project file {projectFile} does not contain existing package or project reference to {packageName}"); + continue; } projectContents = Regex.Replace( @@ -75,16 +94,7 @@ public class Net : LanguageBase } } - File.WriteAllText(projectFile, projectContents); - - Util.DeleteIfExists(PublishDirectory); - - // Disable source link, since it's not needed for perf runs, and also fails in sparse checkout repos - var processArguments = $"publish -c release -f net{languageVersion}.0 -o {PublishDirectory} -p:EnableSourceLink=false {additionalBuildArguments} {project}"; - - var result = await Util.RunAsync("dotnet", processArguments, workingDirectory: WorkingDirectory); - - return (result.StandardOutput, result.StandardError, null); + await File.WriteAllTextAsync(projectFile, projectContents); } public override async Task RunAsync( @@ -187,13 +197,14 @@ public override IDictionary FilterRuntimePackageVersions(IDictio public override Task CleanupAsync(string project) { Util.DeleteIfExists(PublishDirectory); + RestoreBackup(CoreTestFrameworkProjectFile); + RestoreBackup(Path.Combine(WorkingDirectory, project)); + return Task.CompletedTask; + } - var projectFile = Path.Combine(WorkingDirectory, project); - - // Restore backup + private static void RestoreBackup(string projectFile) + { File.Move(projectFile + ".bak", projectFile, overwrite: true); - - return Task.CompletedTask; } } } From 9de28e8cfb92367ccb1d97bb7f868ec77c731958 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 12 Oct 2023 10:55:09 -0700 Subject: [PATCH 60/93] Use better description for single repo sparse checkout (#7106) --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 448cb2c2e31..1f5e3fc375f 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -17,7 +17,10 @@ steps: - checkout: none - task: PowerShell@2 - displayName: 'Sparse checkout repositories' + ${{ if eq(length(parameters.Repositories), 1) }}: + displayName: 'Sparse checkout ${{ parameters.Repositories[0].Name }}' + ${{ else }}: + displayName: 'Sparse checkout repositories' inputs: targetType: inline # Define this inline, because of the chicken/egg problem with loading a script when nothing From 52a573351e9a5e1ae0476349b2880b77e66d819e Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Thu, 12 Oct 2023 16:16:10 -0700 Subject: [PATCH 61/93] [tsp-client] Code fixes (#7100) * fix main * more clean up --- tools/tsp-client/src/index.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index c3ae6673ebd..5e168338841 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -203,17 +203,6 @@ async function generate({ } } -async function syncAndGenerate({ - outputDir, - noCleanup, -}: { - outputDir: string; - noCleanup: boolean; -}) { - await syncTspFiles(outputDir); - await generate({ rootUrl: outputDir, noCleanup}); -} - async function main() { const options = await getOptions(); if (options.debug) { @@ -236,14 +225,15 @@ async function main() { const outputDir = await sdkInit({config: options.tspConfig!, outputDir: rootUrl, emitter, commit: options.commit, repo: options.repo, isUrl: options.isUrl}); Logger.info(`SDK initialized in ${outputDir}`); if (!options.skipSyncAndGenerate) { - await syncAndGenerate({outputDir, noCleanup: options.noCleanup}) + await syncTspFiles(outputDir); + await generate({ rootUrl: outputDir, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); } break; case "sync": - syncTspFiles(rootUrl, options.localSpecRepo); + await syncTspFiles(rootUrl, options.localSpecRepo); break; case "generate": - generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); + await generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); break; case "update": if (options.repo && !options.commit) { @@ -262,7 +252,8 @@ async function main() { repo = tspConfig.repo ?? repo; await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`); } - syncAndGenerate({outputDir: rootUrl, noCleanup: options.noCleanup}); + await syncTspFiles(rootUrl); + await generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); break; default: Logger.error(`Unknown command: ${options.command}`); From 3e64e394d388271e43a51cd1984b099723aca45e Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:25:27 -0400 Subject: [PATCH 62/93] Set product as parent of package work item. (#7075) --- .../Helpers/DevOps-WorkItem-Helpers.ps1 | 77 +------------------ 1 file changed, 2 insertions(+), 75 deletions(-) diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 index ef730282e24..1435e3bb8a4 100644 --- a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 +++ b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 @@ -182,56 +182,6 @@ function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand = return $null } -$releasePlanWorkItems = @{} -function FindReleasePlanWorkItem($serviceName, $packageDisplayName, $outputCommand = $false, $ignoreReleasePlannerTests = $true) -{ - $key = BuildHashKey $serviceName $packageDisplayName - if ($key -and $releasePlanWorkItems.ContainsKey($key)) { - return $releasePlanWorkItems[$key] - } - - if ($serviceName) { - $condition = "[ServiceName] = '${serviceName}'" - if ($packageDisplayName) { - $condition += " AND [PackageDisplayName] = '${packageDisplayName}'" - } - else { - $condition += " AND [PackageDisplayName] = ''" - } - } - else { - $condition = "[ServiceName] <> ''" - } - $condition += " AND [System.State] <> 'Finished'" - if($ignoreReleasePlannerTests){ - $condition += " AND [Tags] NOT CONTAINS 'Release Planner App Test'" - } - - $query = "SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Release Plan' AND ${condition}" - - $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent", "System.Tags") - - $workItems = Invoke-Query $fields $query $outputCommand - - foreach ($wi in $workItems) - { - $localKey = BuildHashKey $wi.fields["Custom.ServiceName"] $wi.fields["Custom.PackageDisplayName"] - if (!$localKey) { continue } - if ($releasePlanWorkItems.ContainsKey($localKey) -and $releasePlanWorkItems[$localKey].id -ne $wi.id) { - Write-Warning "Already found parent [$($releasePlanWorkItems[$localKey].id)] with key [$localKey], using that one instead of [$($wi.id)]." - } - else { - Write-Verbose "[$($wi.id)]$localKey - Cached" - $releasePlanWorkItems[$localKey] = $wi - } - } - - if ($key -and $releasePlanWorkItems.ContainsKey($key)) { - return $releasePlanWorkItems[$key] - } - return $null -} - $packageWorkItems = @{} $packageWorkItemWithoutKeyFields = @{} @@ -545,40 +495,17 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte } } - $newparentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false + $newparentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false UpdateWorkItemParent $existingItem $newParentItem -outputCommand $outputCommand return $existingItem } - $parentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false + $parentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false $workItem = CreateWorkItem $title "Package" "Release" "Release" $fields $assignedTo $parentItem.id -outputCommand $outputCommand Write-Host "[$($workItem.id)]$lang - $pkgName($verMajorMinor) - Created" return $workItem } -function FindOrCreateReleasePlanParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) -{ - $existingItem = FindReleasePlanWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests - if ($existingItem) { - Write-Host "Found existing release plan work item [$($existingItem.id)]" - $newparentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests - UpdateWorkItemParent $existingItem $newParentItem - return $existingItem - } - - $fields = @() - $fields += "`"PackageDisplayName=${packageDisplayName}`"" - $fields += "`"ServiceName=${serviceName}`"" - $productParentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests - $title = "Release Plan - $($packageDisplayName)" - $workItem = CreateWorkItem $title "Release Plan" "Release" "Release" $fields $null $productParentItem.id - - $localKey = BuildHashKey $serviceName $packageDisplayName - Write-Host "[$($workItem.id)]$localKey - Created release plan work item" - $releasePlanWorkItems[$localKey] = $workItem - return $workItem -} - function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true) { $existingItem = FindParentWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests From a7f6104310db72124da540d776a492247ae96786 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 16 Oct 2023 20:07:32 -0400 Subject: [PATCH 63/93] Remove ARM deployment in test resources (#7113) * Resilient blob deletion handling * Remove ARM deployment after deploy --------- Co-authored-by: Heath Stewart --- .../TestResources/New-TestResources.ps1 | 4 ++- eng/scripts/Remove-WormStorageAccounts.ps1 | 7 +++- eng/scripts/live-test-resource-cleanup.ps1 | 36 +++++++++++-------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 49616b64b33..9d940e72316 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -771,7 +771,6 @@ try { -TemplateParameterObject $templateFileParameters ` -Force:$Force } - if ($deployment.ProvisioningState -ne 'Succeeded') { Write-Host "Deployment '$($deployment.DeploymentName)' has state '$($deployment.ProvisioningState)' with CorrelationId '$($deployment.CorrelationId)'. Exiting..." Write-Host @' @@ -803,6 +802,9 @@ try { Write-Verbose "Removing compiled bicep file $($templateFile.jsonFilePath)" Remove-Item $templateFile.jsonFilePath } + + Write-Host "Deleting ARM deployment as it may contain secrets. Deployed resources will not be affected." + $null = $deployment | Remove-AzResourceGroupDeployment } } finally { diff --git a/eng/scripts/Remove-WormStorageAccounts.ps1 b/eng/scripts/Remove-WormStorageAccounts.ps1 index 98ffd0ca46e..6d37365dc39 100644 --- a/eng/scripts/Remove-WormStorageAccounts.ps1 +++ b/eng/scripts/Remove-WormStorageAccounts.ps1 @@ -41,7 +41,12 @@ foreach ($group in $groups) { Write-Error $_ throw } - $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + # Sometimes we get a 404 blob not found but can still delete containers, + # and sometimes we must delete the blob if there's a legal hold. + # Try to remove the blob, but keep running regardless. + try { + $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + } catch {} # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs $ctx | Get-AzStorageContainer | % { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index 2f6a0a8669a..bdbe25ca313 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -52,6 +52,9 @@ param ( [Parameter()] [switch] $DeleteNonCompliantGroups, + [Parameter()] + [switch] $DeleteArmDeployments, + [Parameter()] [int] $DeleteAfterHours = 24, @@ -269,7 +272,7 @@ function FindOrCreateDeleteAfterTag { [object]$ResourceGroup ) - if (!$ResourceGroup) { + if (!$DeleteNonCompliantGroups -or !$ResourceGroup) { return } @@ -326,6 +329,14 @@ function HasDeleteLock([object]$ResourceGroup) { return $false } +function DeleteArmDeployments([object]$ResourceGroup) { + if (!$DeleteArmDeployments) { + return + } + Write-Host "Deleting ARM deployments for group $($ResourceGroup.ResourceGroupName) as they may contain secrets. Deployed resources will not be affected." + $null = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Remove-AzResourceGroupDeployment +} + function DeleteOrUpdateResourceGroups() { [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param() @@ -338,6 +349,7 @@ function DeleteOrUpdateResourceGroups() { [Array]$allGroups = Retry { Get-AzResourceGroup } $toDelete = @() $toUpdate = @() + $toClean = @() Write-Host "Total Resource Groups: $($allGroups.Count)" foreach ($rg in $allGroups) { @@ -351,31 +363,25 @@ function DeleteOrUpdateResourceGroups() { } continue } - if (!$DeleteNonCompliantGroups) { - continue - } - if (HasDoNotDeleteTag $rg) { + if ((IsChildResource $rg) -or (HasDeleteLock $rg)) { continue } - if (IsChildResource $rg) { - continue - } - if (HasValidAliasInName $rg) { - continue - } - if (HasValidOwnerTag $rg) { - continue - } - if (HasDeleteLock $rg) { + if ((HasDoNotDeleteTag $rg) -or (HasValidAliasInName $rg) -or (HasValidOwnerTag $rg)) { + $toClean += $rg continue } $toUpdate += $rg } + foreach ($rg in $toUpdate) { FindOrCreateDeleteAfterTag $rg } + foreach ($rg in $toClean) { + DeleteArmDeployments $rg + } + # Get purgeable resources already in a deleted state. $purgeableResources = @(Get-PurgeableResources) From de734db1e3532dc9f226ba3ca005f69c9e1071b2 Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Mon, 16 Oct 2023 18:42:18 -0700 Subject: [PATCH 64/93] [tsp-client] Fix docs and clean up options (#7099) * clean up unused options * improve docs * option renames * update docs * update info message * update message * remove languageSetting.ts --- tools/tsp-client/src/languageSettings.ts | 61 ------------------------ tools/tsp-client/src/log.ts | 36 ++++++++------ tools/tsp-client/src/options.ts | 42 +++------------- 3 files changed, 28 insertions(+), 111 deletions(-) delete mode 100644 tools/tsp-client/src/languageSettings.ts diff --git a/tools/tsp-client/src/languageSettings.ts b/tools/tsp-client/src/languageSettings.ts deleted file mode 100644 index 1671d8db127..00000000000 --- a/tools/tsp-client/src/languageSettings.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { join } from "node:path"; - -interface LanguageEmitterSettings { - emitterName: string; - emitterOptions: Record; -} - -export const knownLanguages = ["csharp", "java", "javascript", "python", "openapi"] as const; -export type KnownLanguage = (typeof knownLanguages)[number]; -export const languageAliases: Record = { - cs: "csharp", - js: "javascript", - ts: "javascript", - typescript: "javascript", - py: "python", -}; - -export function getEmitterPackage(language: string): string { - if (language in languageEmitterSettings) { - return languageEmitterSettings[language as KnownLanguage].emitterName; - } - throw new Error(`Unknown language ${language}`); -} - -export function getEmitterOutputPath(language: string, projectDirectory: string): string { - if (language === "csharp") { - return join(projectDirectory, "src"); - } - return projectDirectory; -} - -const languageEmitterSettings: Record = { - csharp: { - emitterName: "@azure-tools/typespec-csharp", - emitterOptions: { - "emitter-output-dir": "$projectDirectory/src", - }, - }, - java: { - emitterName: "@azure-tools/typespec-java", - emitterOptions: { - "emitter-output-dir": "$projectDirectory", - }, - }, - javascript: { - emitterName: "@azure-tools/typespec-ts", - emitterOptions: { - "emitter-output-dir": "$projectDirectory", - }, - }, - python: { - emitterName: "@azure-tools/typespec-python", - emitterOptions: { - "emitter-output-dir": "$projectDirectory", - }, - }, - openapi: { - emitterName: "@typespec/openapi3", - emitterOptions: {}, - }, -}; diff --git a/tools/tsp-client/src/log.ts b/tools/tsp-client/src/log.ts index 569364c8438..d3a5e8b5520 100644 --- a/tools/tsp-client/src/log.ts +++ b/tools/tsp-client/src/log.ts @@ -55,26 +55,32 @@ export function printBanner() { } const usageText = ` -Usage: tsp-client [options] +Usage: tsp-client [options] -Generate from a tsp file using --mainFile or use tsp-location.yaml inside -the outputDir. +Use one of the supported commands to get started generating clients from a TypeSpec project. +This tool will default to using your current working directory to generate clients in and will +use it to look for relevant configuration files. To specify a different directory, use +the -o or --output-dir option. -Positionals: +Commands: init Initialize the SDK project folder from a tspconfig.yaml [string] - sync Sync tsp files using tsp-location.yaml [string] - generate Generate from a tsp project [string] - update Sync and generate from a tsp project [string] + sync Sync TypeSpec project specified in tsp-location.yaml [string] + generate Generate from a TypeSpec project [string] + update Sync and generate from a TypeSpec project [string] Options: - -d, --debug Enable debug logging [boolean] - -e, --emitter Which language emitter to use [required] [string] - [choices: "csharp", "java", "javascript", "python", "openapi"] - -m, --mainFile The url of the main tsp file to generate from [string] - --noCleanup Don't clean up the temp directory after generation [boolean] - -h, --help Show help [boolean] - -v, --version Show version number [boolean] - -o, --outputDir The output directory for the emitter + -c, --tsp-config The tspconfig.yaml file to use [string] + --commit Commit to be used for project init or update [string] + -d, --debug Enable debug logging [boolean] + --emitter-options The options to pass to the emitter [string] + -h, --help Show help [boolean] + --local-spec-repo Path to local repository with the TypeSpec project [string] + --save-inputs Don't clean up the temp directory after generation [boolean] + --skip-sync-and-generate Skip sync and generate during project init [boolean] + -o, --output-dir The output directory for the generated files [string] + --repo Repository where the project is defined for init + or update [string] + -v, --version Show version number [boolean] `; export function printUsage() { Logger(usageText); diff --git a/tools/tsp-client/src/options.ts b/tools/tsp-client/src/options.ts index bbefda0a33a..d756054dfd7 100644 --- a/tools/tsp-client/src/options.ts +++ b/tools/tsp-client/src/options.ts @@ -1,7 +1,6 @@ import { parseArgs } from "node:util"; import { Logger, printUsage, printVersion } from "./log.js"; import * as path from "node:path"; -import { knownLanguages, languageAliases } from "./languageSettings.js"; import { doesFileExist } from "./network.js"; import PromptSync from "prompt-sync"; @@ -9,8 +8,6 @@ export interface Options { debug: boolean; command: string; tspConfig?: string; - emitter?: string; - mainFile?: string; outputDir: string; noCleanup: boolean; skipSyncAndGenerate: boolean; @@ -37,29 +34,19 @@ export async function getOptions(): Promise { type: "boolean", short: "d", }, - emitter: { - type: "string", - short: "e", - }, - mainFile: { - type: "string", - short: "m", - }, - outputDir: { + ["output-dir"]: { type: "string", short: "o", }, - tspConfig: { + ["tsp-config"]: { type: "string", short: "c", }, commit: { type: "string", - short: "C", }, repo: { type: "string", - short: "R", }, ["emitter-options"]: { type: "string", @@ -85,19 +72,6 @@ export async function getOptions(): Promise { process.exit(0); } - if (values.emitter) { - let emitter = values.emitter.toLowerCase(); - if (emitter in languageAliases) { - emitter = languageAliases[emitter]!; - } - if (!(knownLanguages as readonly string[]).includes(emitter)) { - Logger.error(`Unknown language ${values.emitter}`); - Logger.error(`Valid languages are: ${knownLanguages.join(", ")}`); - printUsage(); - process.exit(1); - } - } - if (positionals.length === 0) { Logger.error("Command is required"); printUsage(); @@ -112,12 +86,12 @@ export async function getOptions(): Promise { let isUrl = false; if (positionals[0] === "init") { - if (!values.tspConfig) { + if (!values["tsp-config"]) { Logger.error("tspConfig is required"); printUsage(); process.exit(1); } - if (await doesFileExist(values.tspConfig)) { + if (await doesFileExist(values["tsp-config"])) { isUrl = true; } if (!isUrl) { @@ -130,8 +104,8 @@ export async function getOptions(): Promise { } // By default, assume that the command is run from the output directory let outputDir = "."; - if (values.outputDir) { - outputDir = values.outputDir; + if (values["output-dir"]) { + outputDir = values["output-dir"]; } outputDir = path.resolve(path.normalize(outputDir)); @@ -153,9 +127,7 @@ export async function getOptions(): Promise { return { debug: values.debug ?? false, command: positionals[0], - tspConfig: values.tspConfig, - emitter: values.emitter, - mainFile: values.mainFile, + tspConfig: values["tsp-config"], noCleanup: values["save-inputs"] ?? false, skipSyncAndGenerate: values["skip-sync-and-generate"] ?? false, outputDir: outputDir, From ec3ec7571e0a4ca9753e1aa7ea77813ebf207a89 Mon Sep 17 00:00:00 2001 From: Libba Lawrence Date: Tue, 17 Oct 2023 09:06:20 -0700 Subject: [PATCH 65/93] [pylint] Type format incorrect check (#7024) * type misformatted * update checker * update script * check diff incorrect types * add to readme * typo in readme * typo2 * make class list * specify * update * order of param * switch back * fix * update readme * remove re import * update format * forgot space * correct test * remove raises * raises * order --- .../azure-pylint-guidelines-checker/README.md | 1 + .../pylint_guidelines_checker.py | 64 ++++++++++++++----- .../tests/test_pylint_custom_plugins.py | 55 +++++++++++++++- 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md b/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md index fb9e5caf308..39ae46f3fae 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/README.md @@ -90,3 +90,4 @@ docstring-should-be-keyword | pylint:disable=docstring-should-be-keyword | do-not-import-legacy-six | pylint:disable=do-not-import-legacy-six | Do not import six. | No Link. | no-legacy-azure-core-http-response-import | pylint:disable=no-legacy-azure-core-http-response-import | Do not import HttpResponse from azure.core.pipeline.transport outside of Azure Core. You can import HttpResponse from azure.core.rest instead. | [link](https://github.com/Azure/azure-sdk-for-python/issues/30785) | docstring-keyword-should-match-keyword-only | pylint:disable=docstring-keyword-should-match-keyword-only | Docstring keyword arguments and keyword-only method arguments should match. | [link](https://azure.github.io/azure-sdk/python_documentation.html#docstrings) | +docstring-type-do-not-use-class | pylint:disable=docstring-type-do-not-use-class | Docstring type is formatted incorrectly. Do not use `:class` in docstring type. | [link](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html) | diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py index ad01fb5b91a..45841c82be5 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/pylint_guidelines_checker.py @@ -1234,6 +1234,12 @@ class CheckDocstringParameters(BaseChecker): "docstring-keyword-should-match-keyword-only", "Docstring keyword arguments and keyword-only method arguments should match.", ), + "C4759": ( + '"%s" type formatted incorrectly. Do not use `:class` in docstring type. See details: ' + 'https://azure.github.io/azure-sdk/python_documentation.html#docstrings', + "docstring-type-do-not-use-class", + "Docstring type is formatted incorrectly. Do not use `:class` in docstring type.", + ), } options = ( ( @@ -1283,10 +1289,11 @@ class CheckDocstringParameters(BaseChecker): ), ) + def __init__(self, linter=None): super(CheckDocstringParameters, self).__init__(linter) - def _find_keyword(self, line): + def _find_keyword(self, line, docstring, idx, keyword_args): keyword_args = {} # this param has its type on a separate line if line.startswith("keyword") and line.count(" ") == 1: @@ -1301,6 +1308,10 @@ def _find_keyword(self, line): param = line.split(" ")[-1] param_type = ("").join(line.split(" ")[1:-1]) keyword_args[param] = param_type + if line.startswith("paramtype"): + param = line.split("paramtype ")[1] + if param in keyword_args: + keyword_args[param] = docstring[idx+1] return keyword_args @@ -1358,8 +1369,10 @@ def check_parameters(self, node): vararg_name = node.args.vararg try: + # check for incorrect type :class to prevent splitting + docstring = node.doc.replace(":class:", "CLASS ") # not every method will have a docstring so don't crash here, just return - docstring = node.doc.split(":") + docstring = docstring.split(":") except AttributeError: return @@ -1371,7 +1384,7 @@ def check_parameters(self, node): docstring_keyword_args = {} for idx, line in enumerate(docstring): # check for keyword args in docstring - docstring_keyword_args.update(self._find_keyword(line)) + docstring_keyword_args.update(self._find_keyword(line, docstring, idx, docstring_keyword_args)) # check for params in docstring docparams.update(self._find_param(line, docstring, idx, docparams)) @@ -1397,6 +1410,15 @@ def check_parameters(self, node): msgid="docstring-keyword-should-match-keyword-only", args=(", ".join(missing_kwonly_args)), node=node, confidence=None ) + # check that all types are formatted correctly + add_keyword_type_warnings = [keyword for keyword, doc_type in docstring_keyword_args.items() if doc_type and "CLASS" in doc_type] + if len(add_keyword_type_warnings) > 0: + self.add_message(msgid="docstring-type-do-not-use-class", args=(", ".join(add_keyword_type_warnings)), node=node, confidence=None) + + add_docparams_type_warnings = [param for param, doc_type in docparams.items() if doc_type and "CLASS" in doc_type] + if len(add_docparams_type_warnings) > 0: + self.add_message(msgid="docstring-type-do-not-use-class", args=(", ".join(add_docparams_type_warnings)), node=node, confidence=None) + # check if we have a type for each param and check if documented params that should be keywords missing_types = [] should_be_keywords = [] @@ -1426,29 +1448,41 @@ def check_return(self, node): :param node: ast.FunctionDef :return: None """ - # Get decorators on the function - function_decorators = node.decoratornames() - try: - returns = next(node.infer_call_result()).as_string() - # If returns None ignore - if returns == "None": - return - except (astroid.exceptions.InferenceError, AttributeError): - # this function doesn't return anything, just return - return + + # Check docstring documented returns/raises try: + # check for incorrect type :class to prevent splitting + docstring = node.doc.replace(":class:", "CLASS ") # not every method will have a docstring so don't crash here, just return - docstring = node.doc.split(":") + docstring = docstring.split(":") except AttributeError: return - + has_return, has_rtype = False, False for line in docstring: if line.startswith("return"): has_return = True if line.startswith("rtype"): has_rtype = True + try: + if "CLASS" in docstring[docstring.index("rtype") + 1]: + self.add_message( + msgid="docstring-type-do-not-use-class", args="rtype", node=node, confidence=None + ) + except: + pass + + # Get decorators on the function + function_decorators = node.decoratornames() + try: + returns = next(node.infer_call_result()).as_string() + # If returns None ignore + if returns == "None": + return + except (astroid.exceptions.InferenceError, AttributeError): + # this function doesn't return anything, just return + return # If is an @property decorator, don't require :return: as it is repetitive if has_return is False and "builtins.property" not in function_decorators: diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py b/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py index 99c3a35a69c..8bdc58bc1e0 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/tests/test_pylint_custom_plugins.py @@ -3426,7 +3426,7 @@ def function_foo(*x, y, z): msg_id='docstring-keyword-should-match-keyword-only', line=2, node=node, - args='z, y', + args='y, z', col_offset=0, end_line=2, end_col_offset=16 @@ -3548,6 +3548,59 @@ def function_foo(self, x, *, z, y=None): with self.assertNoMessages(): self.checker.visit_functiondef(node) + def test_docstring_correct_rtype(self): + node = astroid.extract_node( + """ + def function_foo(self, x, *, z, y=None) -> str: + ''' + :param x: x + :type x: str + :keyword str y: y + :keyword str z: z + :rtype: str + ''' + print("hello") + """ + ) + with self.assertNoMessages(): + self.checker.visit_functiondef(node) + + def test_docstring_class_type(self): + node = astroid.extract_node( + """ + def function_foo(self, x, y): + ''' + :param x: x + :type x: :class:`azure.core.credentials.AccessToken` + :param y: y + :type y: str + :rtype: :class:`azure.core.credentials.AccessToken` + ''' + print("hello") + """ + ) + with self.assertAddsMessages( + pylint.testutils.MessageTest( + msg_id="docstring-type-do-not-use-class", + line=2, + args="x", + node=node, + col_offset=0, + end_line=3, + end_col_offset=16 + ), + pylint.testutils.MessageTest( + msg_id="docstring-type-do-not-use-class", + line=2, + args="rtype", + node=node, + col_offset=0, + end_line=2, + end_col_offset=16 + ), + ): + self.checker.visit_functiondef(node) + class TestDoNotImportLegacySix(pylint.testutils.CheckerTestCase): """Test that we are blocking disallowed imports and allowing allowed imports.""" From f9d251aca7dc5b4aed3fac01f0a77f2f5ee6f3da Mon Sep 17 00:00:00 2001 From: Libba Lawrence Date: Tue, 17 Oct 2023 10:06:58 -0700 Subject: [PATCH 66/93] bump version (#7139) --- .../azure-pylint-guidelines-checker/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md index 5e20c9dbce8..59b5ad9d7d9 100644 --- a/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md +++ b/tools/pylint-extensions/azure-pylint-guidelines-checker/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 0.2.0 (Unreleased) +## 0.2.0 (2023-10-17) - Checker to enforce docstring keywords being keyword-only in method signature. - Fixed a bug in `no-legacy-azure-core-http-response-import` that was throwing warnings for azure-mgmt-core. From 2efe0b2298729cd58066bb2866adb0a588532f77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:23:50 -0700 Subject: [PATCH 67/93] Bump Azure.Identity (#7158) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.8.2 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.8.2...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Azure.Sdk.Tools.AccessManagement.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/secret-management/Azure.Sdk.Tools.AccessManagement/Azure.Sdk.Tools.AccessManagement.csproj b/tools/secret-management/Azure.Sdk.Tools.AccessManagement/Azure.Sdk.Tools.AccessManagement.csproj index 6bde472c1cd..5a5cf6c8d01 100644 --- a/tools/secret-management/Azure.Sdk.Tools.AccessManagement/Azure.Sdk.Tools.AccessManagement.csproj +++ b/tools/secret-management/Azure.Sdk.Tools.AccessManagement/Azure.Sdk.Tools.AccessManagement.csproj @@ -1,7 +1,7 @@ - + From a114a07e255b02f662bb0c06661e7d388da2b511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:46:25 -0700 Subject: [PATCH 68/93] Bump Azure.Identity (#7157) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.8.0 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.8.0...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ....Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory.csproj b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory.csproj index a143fb620c6..c0e68d55e28 100644 --- a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory.csproj +++ b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory/Azure.Sdk.Tools.SecretRotation.Stores.AzureActiveDirectory.csproj @@ -1,7 +1,7 @@ - + From 985663185284a2e5ee4940993d86958abe93085f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:15:54 -0700 Subject: [PATCH 69/93] Bump Azure.Identity in /tools/github-issues/Azure.Sdk.Tools.GitHubIssues (#7151) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.2.3 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.2.3...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Azure.Sdk.Tools.GitHubIssues.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/github-issues/Azure.Sdk.Tools.GitHubIssues/Azure.Sdk.Tools.GitHubIssues.csproj b/tools/github-issues/Azure.Sdk.Tools.GitHubIssues/Azure.Sdk.Tools.GitHubIssues.csproj index de7c2b144e5..38a2c2b3cf6 100644 --- a/tools/github-issues/Azure.Sdk.Tools.GitHubIssues/Azure.Sdk.Tools.GitHubIssues.csproj +++ b/tools/github-issues/Azure.Sdk.Tools.GitHubIssues/Azure.Sdk.Tools.GitHubIssues.csproj @@ -7,7 +7,7 @@ - + From 171966ed1e30d2918c4785ca56a07daf8ecb6e0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:16:13 -0700 Subject: [PATCH 70/93] Bump Azure.Identity (#7152) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.2.3 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.2.3...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Azure.Sdk.Tools.WebhookRouter.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/webhook-router/Azure.Sdk.Tools.WebhookRouter/Azure.Sdk.Tools.WebhookRouter.csproj b/tools/webhook-router/Azure.Sdk.Tools.WebhookRouter/Azure.Sdk.Tools.WebhookRouter.csproj index 62ba5e608a6..8016b910748 100644 --- a/tools/webhook-router/Azure.Sdk.Tools.WebhookRouter/Azure.Sdk.Tools.WebhookRouter.csproj +++ b/tools/webhook-router/Azure.Sdk.Tools.WebhookRouter/Azure.Sdk.Tools.WebhookRouter.csproj @@ -1,11 +1,11 @@ - + net6.0 v3 - + From f4efb4332616bcaeebbfeea56ce77916f7dc1a4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:16:30 -0700 Subject: [PATCH 71/93] Bump Azure.Identity from 1.5.0 to 1.10.2 in /tools/identity-resolution (#7153) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.5.0 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.5.0...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/identity-resolution/identity-resolution.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/identity-resolution/identity-resolution.csproj b/tools/identity-resolution/identity-resolution.csproj index f8666f676aa..61e2bd463bb 100644 --- a/tools/identity-resolution/identity-resolution.csproj +++ b/tools/identity-resolution/identity-resolution.csproj @@ -10,7 +10,7 @@ - + From 83a363cf8463c2cdf6a20e385916115db1c81e3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:16:56 -0700 Subject: [PATCH 72/93] Bump Azure.Identity (#7155) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.8.0 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.8.0...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Azure.Sdk.Tools.SecretRotation.Stores.KeyVault.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault.csproj b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault.csproj index 8c62f2d4bfd..f5cafc4a976 100644 --- a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault.csproj +++ b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault/Azure.Sdk.Tools.SecretRotation.Stores.KeyVault.csproj @@ -1,7 +1,7 @@ - + - + From 3f21000edef991f119cb96949a4cd54c2afac6f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:18:33 -0700 Subject: [PATCH 73/93] Bump Azure.Identity (#7156) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.8.0 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.8.0...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Azure.Sdk.Tools.SecretRotation.Azure.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Azure/Azure.Sdk.Tools.SecretRotation.Azure.csproj b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Azure/Azure.Sdk.Tools.SecretRotation.Azure.csproj index c825ef67ae2..f293f10c821 100644 --- a/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Azure/Azure.Sdk.Tools.SecretRotation.Azure.csproj +++ b/tools/secret-management/Azure.Sdk.Tools.SecretRotation.Azure/Azure.Sdk.Tools.SecretRotation.Azure.csproj @@ -2,7 +2,7 @@ - + From 62725249a5bfcafe89072f7cd0906dcbc9e3d02f Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 19 Oct 2023 14:27:43 -0700 Subject: [PATCH 74/93] Add CodeownersUtils, a CodeownersParser replacement (#7097) * Add CodeownersUtils * Add pipeline yml * Add README and METADATA markdown files * Use string insensitive comparison when checking the Azure org for team entries * Remove nonexistent tools/check-enforcer path entry and add tools/codeowners-parser with owners * Updates based upon feedback, minor changes for AzureSdkOwner being tied to ServiceLabel and updates to remove block formatting errors from filtering. * Update readme for changes in the previous commit --- .github/CODEOWNERS | 2 +- ...ure.Sdk.Tools.CodeownersUtils.Tests.csproj | 35 ++ .../CodeownersTestFiles/Baseline/NoErrors | 31 ++ .../Baseline/NoErrors_EmptyBaseline.txt | 0 .../Baseline/WithBlockErrors | 65 +++ .../Baseline/WithBlockErrors_FullBaseline.txt | 5 + .../CodeownersTestFiles/EndToEnd/NoErrors | 34 ++ .../EndToEnd/NoErrorsExpectedEntries.json | 130 ++++++ .../EndToEnd/WithBlockErrors | 58 +++ .../WithBlockErrorsExpectedEntries.json | 82 ++++ .../EndToEnd/WithBlockErrors_baseline.txt | 5 + .../CodeownersTestFiles/EndToEnd/WithErrors | 64 +++ .../EndToEnd/WithErrors_baseline.txt | 5 + .../AllMonikersEndsWithSourcePath | 6 + .../FindBlockEnd/PRLabelAndSourcePath | 3 + .../FindBlockEnd/ServiceLabelAndMissingFolder | 3 + .../FindBlockEnd/ServiceLabelAndSourcePath | 3 + .../FindBlockEnd/SingleSourcePathLine | 1 + .../Matching/SubDirAndFallback | 27 ++ .../VerifyBlock/AzureSdkOwnersAndServiceLabel | 11 + .../VerifyBlock/DuplicateMonikers | 37 ++ .../VerifyBlock/MonikersEndsInSourcePath | 5 + .../VerifyBlock/MonikersMissingSourcePath | 19 + .../VerifyBlock/PRLabelAndSourcePath | 3 + .../VerifyBlock/ServiceLabelAndMissingPath | 3 + .../VerifyBlock/ServiceLabelAndServiceOwners | 3 + .../VerifyBlock/ServiceLabelAndSourcePath | 3 + .../ServiceLabelTooManyOwnersAndMonikers | 16 + .../VerifyBlock/SingleSourcePathLine | 2 + .../Mocks/DirectoryUtilsMock.cs | 26 ++ .../Parsing/CodeownersParserTests.cs | 98 +++++ .../TestHelpers.cs | 197 +++++++++ .../Utils/BaselineUtilsTests.cs | 149 +++++++ .../Utils/DirectoryUtilsTests.cs | 255 ++++++++++++ .../Utils/MonikerUtilsTests.cs | 71 ++++ .../Utils/OwnerDataUtilsTests.cs | 126 ++++++ .../Utils/ParsingUtilsTests.cs | 281 +++++++++++++ .../Utils/RepoLabelDataUtilsTests.cs | 64 +++ .../Verification/CodeownersLinterTests.cs | 375 ++++++++++++++++++ .../Verification/LabelsTests.cs | 110 +++++ .../Verification/OwnersTests.cs | 92 +++++ .../Azure.Sdk.Tools.CodeownersLinter.csproj | 18 + .../Program.cs | 231 +++++++++++ .../Azure.Sdk.Tools.CodeownersUtils.csproj | 13 + .../Caches/RepoLabelCache.cs | 66 +++ .../Caches/TeamUserCache.cs | 86 ++++ .../Caches/UserOrgVisibilityCache.cs | 60 +++ .../Constants/BaselineConstants.cs | 16 + .../Constants/DefaultStorageConstants.cs | 18 + .../Constants/ErrorMessageConstants.cs | 66 +++ .../Constants/GlobConstants.cs | 49 +++ .../Constants/LabelConstants.cs | 16 + .../Constants/MonikerConstants.cs | 23 ++ .../Constants/OrgConstants.cs | 18 + .../Constants/SeparatorConstants.cs | 18 + .../Errors/BaseError.cs | 50 +++ .../Errors/BlockFormattingError.cs | 58 +++ .../Errors/SingleLineError.cs | 42 ++ .../Parsing/CodeownersEntry.cs | 171 ++++++++ .../Parsing/CodeownersParser.cs | 219 ++++++++++ .../Utils/BaselineUtils.cs | 125 ++++++ .../Utils/DirectoryUtils.cs | 277 +++++++++++++ .../Utils/FileHelpers.cs | 92 +++++ .../Utils/MonikerUtils.cs | 86 ++++ .../Utils/OwnerDataUtils.cs | 118 ++++++ .../Utils/ParsingUtils.cs | 231 +++++++++++ .../Utils/RepoLabelDataUtils.cs | 53 +++ .../Verification/CodeownersLinter.cs | 325 +++++++++++++++ .../Verification/Labels.cs | 59 +++ .../Verification/Owners.cs | 96 +++++ tools/codeowners-utils/CodeownersUtils.sln | 37 ++ tools/codeowners-utils/METADATA.md | 93 +++++ tools/codeowners-utils/README.md | 111 ++++++ tools/codeowners-utils/ci.yml | 24 ++ 74 files changed, 5469 insertions(+), 1 deletion(-) create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Azure.Sdk.Tools.CodeownersUtils.Tests.csproj create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors_EmptyBaseline.txt create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors_FullBaseline.txt create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrors create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrorsExpectedEntries.json create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrorsExpectedEntries.json create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors_baseline.txt create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors_baseline.txt create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/AllMonikersEndsWithSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/PRLabelAndSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndMissingFolder create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/SingleSourcePathLine create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Matching/SubDirAndFallback create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/DuplicateMonikers create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersEndsInSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/PRLabelAndSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndMissingPath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndServiceOwners create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndSourcePath create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/SingleSourcePathLine create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Mocks/DirectoryUtilsMock.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Parsing/CodeownersParserTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/TestHelpers.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/BaselineUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/DirectoryUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/MonikerUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/OwnerDataUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/ParsingUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/RepoLabelDataUtilsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/CodeownersLinterTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/LabelsTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/OwnersTests.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Azure.Sdk.Tools.CodeownersLinter.csproj create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Program.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/RepoLabelCache.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/TeamUserCache.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/UserOrgVisibilityCache.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/BaselineConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/DefaultStorageConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/ErrorMessageConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/GlobConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/LabelConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/MonikerConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/OrgConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/SeparatorConstants.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BaseError.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BlockFormattingError.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/SingleLineError.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersEntry.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersParser.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/BaselineUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/DirectoryUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/FileHelpers.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/MonikerUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/OwnerDataUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/ParsingUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/RepoLabelDataUtils.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/CodeownersLinter.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Labels.cs create mode 100644 tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Owners.cs create mode 100644 tools/codeowners-utils/CodeownersUtils.sln create mode 100644 tools/codeowners-utils/METADATA.md create mode 100644 tools/codeowners-utils/README.md create mode 100644 tools/codeowners-utils/ci.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 965467fb175..1297d19391d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,8 +30,8 @@ # Eng Sys Tools ########### /tools/ @azure/azure-sdk-eng -/tools/check-enforcer/ @praveenkuttappan @weshaggard /tools/code-owners-parser/ @konrad-jamrozik @JimSuplizio +/tools/codeowners-utils/ @JimSuplizio /tools/github-issues/ @praveenkuttappan @weshaggard /tools/github-event-processor/ @JimSuplizio @benbp /tools/github-team-user-store/ @JimSuplizio @weshaggard diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Azure.Sdk.Tools.CodeownersUtils.Tests.csproj b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Azure.Sdk.Tools.CodeownersUtils.Tests.csproj new file mode 100644 index 00000000000..74dd145b3bc --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Azure.Sdk.Tools.CodeownersUtils.Tests.csproj @@ -0,0 +1,35 @@ + + + + net6.0 + disable + disable + + false + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + + diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors new file mode 100644 index 00000000000..2770516809e --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors @@ -0,0 +1,31 @@ +# CODEOWNERS with no errors, for end to end tesing. + +# A single source path/owner line +/sdk/someFakePath1 @TestOwner0 + +# PPLabel block ending in a source path/owner line +# PRLabel: TestLabel1 +/sdk/someFakePath2 @TestOwner2 + +# ServiceLabel block ending in a source path/owner line +# ServiceLabel: %TestLabel1 +/sdk/someFakePath3 @TestOwner4 @TestOwner2 + +# ServiceLabel block with MissingFolder moniker for owners +# ServiceLabel: %TestLabel2 +#// @TestOwner0 @TestOwner4 + +# ServiceLabel block with ServiceOwners moniker, no % before label +# ServiceLabel: TestLabel3 +# ServiceOwners: @TestOwner0 @TestOwner2 + +# AzureSdkOwners must be part of a block with a ServiceLabel entry +# AzureSdkOwners: @TestOwner0 +# ServiceLabel: %TestLabel3 +/sdk/someFakePath4 @TestOwner2 @TestOwner4 + +# Every moniker that can be grouped together and end in a source path/owner line +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath5 @TestOwner2 @TestOwner4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors_EmptyBaseline.txt b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/NoErrors_EmptyBaseline.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors new file mode 100644 index 00000000000..2c4dff7625d --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors @@ -0,0 +1,65 @@ +# CODEOWNERS with errors, for end to end tesing. +# Note: For the generated owners, for testing, every odd owner isn't public + +# This block has errors +# 1. TestOwner1 is non-public +# 2. TestOwner42 doesn't exist +# owner that doesn't exist +/sdk/someFakePath1 @TestOwner1 @TestOwner2 @TestOwner42 + +# This block does not have errors. +# PRLabel block ending in a source path/owner line +# PRLabel: %TestLabel1 +/sdk/someFakePath2 @TestOwner2 + +# This block has errors +# 1. The label won't exist for the repository +# 2. The PRLabel moniker needs to be in a block that ends in a source path/owner line +# PRLabel: %TestLabel987 + +# This block has no errors +# ServiceLabel block with MissingFolder moniker for owners +# ServiceLabel: %TestLabel2 +#// @TestOwner0 @TestOwner4 + +# This block has errors +# 1. TestLabel55 doesn't exist for the repository +# 2. TestOwner3 isn't a public member of Azure +# ServiceLabel block with ServiceOwners moniker +# ServiceLabel: %TestLabel55 +# ServiceOwners: @TestOwner0 @TestOwner3 + +# This block has no errors +# AzureSdkOwners must be part of a block with a ServiceLabel entry +# AzureSdkOwners: @TestOwner0 +# ServiceLabel: %TestLabel55 +/sdk/someFakePath4 @TestOwner2 @TestOwner4 + +# This block has errors +# AzureSdkOwners must be part of a block with a ServiceLabel entry and block +# is missing the ServiceLabel entry +# AzureSdkOwners: @TestOwner0 +# ServiceOwners: @TestOwner2 + +# This block has errors +# 1. AzureSdkOwners exists twice in the same block +# 2. TestOwner3 isn't a public member of Azure +# AzureSdkOwners: @TestOwner2 +# AzureSdkOwners: @TestOwner3 +/sdk/someFakePath5 @TestOwner2 @TestOwner4 + +# This block has errors +# 1. ServiceLabel needs to be part of a block that has ServiceOwners or ends +# in a source path/owner line but not both +# 2. The TestOwner1 isn't a public member of Azure +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner1 +/sdk/someFakePath6 @TestOwner2 @TestOwner4 + +# This block does not have errors +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath7 @TestOwner2 @TestOwner4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors_FullBaseline.txt b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors_FullBaseline.txt new file mode 100644 index 00000000000..73e89f3dd89 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Baseline/WithBlockErrors_FullBaseline.txt @@ -0,0 +1,5 @@ +TestOwner1 is not a public member of Azure. +TestOwner42 is an invalid user. Ensure the user exists, is public member of Azure and has write permissions. +'TestLabel987' is not a valid label for this repository. +'TestLabel55' is not a valid label for this repository. +TestOwner3 is not a public member of Azure. diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrors b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrors new file mode 100644 index 00000000000..93ef0fed6d4 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrors @@ -0,0 +1,34 @@ +# CODEOWNERS with no errors, for end to end tesing. + +# The fallback/catch all +/** @TestOwner2 + +# A single source path/owner line +/sdk/someFakePath1/ @TestOwner0 + +# PPLabel block ending in a source path/owner line +# PRLabel: TestLabel1 +/sdk/someFakePath2/ @TestOwner2 + +# ServiceLabel block ending in a source path/owner line +# ServiceLabel: %TestLabel1 +/sdk/someFakePath3/ @TestOwner4 @TestOwner2 + +# ServiceLabel block with MissingFolder moniker for owners +# ServiceLabel: %TestLabel2 +#// @TestOwner0 @TestOwner4 + +# ServiceLabel block with ServiceOwners moniker, no % before label +# ServiceLabel: TestLabel3 +# ServiceOwners: @TestOwner0 @TestOwner2 + +# AzureSdkOwners must be part of a block with a ServiceLabel entry +# AzureSdkOwners: @TestOwner0 +# ServiceLabel: %TestLabel3 +/sdk/someFakePath4/ @TestOwner2 @TestOwner4 + +# Every moniker that can be grouped together and end in a source path/owner line. +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath5/ @TestOwner2 @TestOwner4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrorsExpectedEntries.json b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrorsExpectedEntries.json new file mode 100644 index 00000000000..5103586bae7 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/NoErrorsExpectedEntries.json @@ -0,0 +1,130 @@ +[ + { + "PathExpression": "/**", + "ContainsWildcard": true, + "SourceOwners": [ + "TestOwner2" + ], + "PRLabels": [], + "ServiceLabels": [], + "ServiceOwners": [], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "/sdk/someFakePath1/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner0" + ], + "PRLabels": [], + "ServiceLabels": [], + "ServiceOwners": [], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "/sdk/someFakePath2/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner2" + ], + "PRLabels": [ + "TestLabel1" + ], + "ServiceLabels": [], + "ServiceOwners": [], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "/sdk/someFakePath3/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner4", + "TestOwner2" + ], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel1" + ], + "ServiceOwners": [ + "TestOwner4", + "TestOwner2" + ], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "", + "ContainsWildcard": false, + "SourceOwners": [], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel2" + ], + "ServiceOwners": [ + "TestOwner0", + "TestOwner4" + ], + "AzureSdkOwners": [], + "IsValid": false + }, + { + "PathExpression": "", + "ContainsWildcard": false, + "SourceOwners": [], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel3" + ], + "ServiceOwners": [ + "TestOwner0", + "TestOwner2" + ], + "AzureSdkOwners": [], + "IsValid": false + }, + { + "PathExpression": "/sdk/someFakePath4/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel3" + ], + "ServiceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "AzureSdkOwners": [ + "TestOwner0" + ], + "IsValid": true + }, + { + "PathExpression": "/sdk/someFakePath5/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "PRLabels": [ + "TestLabel2" + ], + "ServiceLabels": [ + "TestLabel4" + ], + "ServiceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "AzureSdkOwners": [ + "TestOwner0" + ], + "IsValid": true + } +] diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors new file mode 100644 index 00000000000..d9f26d9b6c3 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors @@ -0,0 +1,58 @@ +# CODEOWNERS with errors, for end to end tesing. +# Note: For the generated owners, for testing, every odd owner isn't public + +# This block has line errors. Linting will find the errors but parsing should still parse the entry. +# 1. TestOwner1 is non-public +# 2. TestOwner42 doesn't exist +# owner that doesn't exist +/sdk/someFakePath1/ @TestOwner1 @TestOwner2 @TestOwner42 + +# This block does not have errors. +# PRLabel block ending in a source path/owner line +# PRLabel: %TestLabel1 +/sdk/someFakePath2/ @TestOwner2 + +# This block has block and line errors +# 1. The label won't exist for the repository +# 2. The PRLabel moniker needs to be in a block that ends in a source path/owner line +# PRLabel: %TestLabel987 + +# This block has no errors +# ServiceLabel block with MissingFolder moniker for owners +# ServiceLabel: %TestLabel2 +#// @TestOwner0 @TestOwner4 + +# This block has line errors. Linting will find the errors but parsing should still parse the entry. +# 1. TestLabel55 doesn't exist for the repository +# 2. TestOwner3 isn't a public member of Azure +# ServiceLabel block with ServiceOwners moniker +# ServiceLabel: %TestLabel55 +# ServiceOwners: @TestOwner0 @TestOwner3 + +# This block has a block error +# AzureSdkOwners must be part of a block with a ServiceLabel +# AzureSdkOwners: @TestOwner0 +/sdk/someFakePath4/ @TestOwner2 @TestOwner4 + +# This block has both block and line errors +# 1. AzureSdkOwners exists twice in the same block +# 2. TestOwner3 isn't a public member of Azure +# AzureSdkOwners: @TestOwner2 +# AzureSdkOwners: @TestOwner3 +/sdk/someFakePath5/ @TestOwner2 @TestOwner4 + +# This block has block and line errors +# 1. ServiceLabel needs to be part of a block that has ServiceOwners or ends +# in a source path/owner line but not both +# 2. The TestOwner1 isn't a public member of Azure +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner1 +/sdk/someFakePath6/ @TestOwner2 @TestOwner4 + +# This block does not have errors +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath7/ @TestOwner2 @TestOwner4 \ No newline at end of file diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrorsExpectedEntries.json b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrorsExpectedEntries.json new file mode 100644 index 00000000000..bf58192c071 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrorsExpectedEntries.json @@ -0,0 +1,82 @@ +[ + { + "PathExpression": "/sdk/someFakePath1/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner1", + "TestOwner2", + "TestOwner42" + ], + "PRLabels": [], + "ServiceLabels": [], + "ServiceOwners": [], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "/sdk/someFakePath2/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner2" + ], + "PRLabels": [ + "TestLabel1" + ], + "ServiceLabels": [], + "ServiceOwners": [], + "AzureSdkOwners": [], + "IsValid": true + }, + { + "PathExpression": "", + "ContainsWildcard": false, + "SourceOwners": [], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel2" + ], + "ServiceOwners": [ + "TestOwner0", + "TestOwner4" + ], + "AzureSdkOwners": [], + "IsValid": false + }, + { + "PathExpression": "", + "ContainsWildcard": false, + "SourceOwners": [], + "PRLabels": [], + "ServiceLabels": [ + "TestLabel55" + ], + "ServiceOwners": [ + "TestOwner0", + "TestOwner3" + ], + "AzureSdkOwners": [], + "IsValid": false + }, + { + "PathExpression": "/sdk/someFakePath7/", + "ContainsWildcard": false, + "SourceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "PRLabels": [ + "TestLabel2" + ], + "ServiceLabels": [ + "TestLabel4" + ], + "ServiceOwners": [ + "TestOwner2", + "TestOwner4" + ], + "AzureSdkOwners": [ + "TestOwner0" + ], + "IsValid": true + } +] \ No newline at end of file diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors_baseline.txt b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors_baseline.txt new file mode 100644 index 00000000000..73e89f3dd89 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithBlockErrors_baseline.txt @@ -0,0 +1,5 @@ +TestOwner1 is not a public member of Azure. +TestOwner42 is an invalid user. Ensure the user exists, is public member of Azure and has write permissions. +'TestLabel987' is not a valid label for this repository. +'TestLabel55' is not a valid label for this repository. +TestOwner3 is not a public member of Azure. diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors new file mode 100644 index 00000000000..6b803fae58d --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors @@ -0,0 +1,64 @@ +# CODEOWNERS with errors, for end to end tesing. +# Note: For the generated owners, for testing, every odd owner isn't public + +# This block has line errors. Linting will find the errors but parsing should still parse the entry. +# 1. TestOwner1 is non-public +# 2. TestOwner42 doesn't exist +# owner that doesn't exist +/sdk/someFakePath1/ @TestOwner1 @TestOwner2 @TestOwner42 + +# This block does not have errors. +# PRLabel block ending in a source path/owner line +# PRLabel: %TestLabel1 +/sdk/someFakePath2/ @TestOwner2 + +# This block has a line error +# 1. The label won't exist for the repository +# PRLabel: %TestLabel987 +/sdk/someFakePath3 @TestOwner4 + +# This block has no errors +# ServiceLabel block with MissingFolder moniker for owners +# ServiceLabel: %TestLabel2 +#// @TestOwner0 @TestOwner4 + +# This block has line errors. Linting will find the errors but parsing should still parse the entry. +# 1. TestLabel55 doesn't exist for the repository +# 2. TestOwner3 isn't a public member of Azure +# ServiceLabel block with ServiceOwners moniker +# ServiceLabel: %TestLabel55 +# ServiceOwners: @TestOwner0 @TestOwner3 + +# This block has no errors +# AzureSdkOwners: @TestOwner0 +# ServiceLabel: %TestLabel3 +# ServiceOwners: @TestOwner2 @TestOwner4 + +# This block has errors +# AzureSdkOwners must be part of a block that contains a ServiceLabel entry +# AzureSdkOwners: @TestOwner0 +/sdk/someFakePath4/ @TestOwner2 @TestOwner4 + +# This block has both block and line errors +# 1. AzureSdkOwners exists twice in the same block +# 2. TestOwner3 isn't a public member of Azure +# AzureSdkOwners: @TestOwner2 +# AzureSdkOwners: @TestOwner3 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath5/ @TestOwner2 @TestOwner4 + +# This block has block and line errors +# 1. ServiceLabel needs to be part of a block that has ServiceOwners or ends +# in a source path/owner line but not both +# 2. The TestOwner1 isn't a public member of Azure +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner1 +/sdk/someFakePath6/ @TestOwner2 @TestOwner4 + +# This block does not have errors +# AzureSdkOwners: @TestOwner0 +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel4 +/sdk/someFakePath7/ @TestOwner2 @TestOwner4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors_baseline.txt b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors_baseline.txt new file mode 100644 index 00000000000..73e89f3dd89 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/EndToEnd/WithErrors_baseline.txt @@ -0,0 +1,5 @@ +TestOwner1 is not a public member of Azure. +TestOwner42 is an invalid user. Ensure the user exists, is public member of Azure and has write permissions. +'TestLabel987' is not a valid label for this repository. +'TestLabel55' is not a valid label for this repository. +TestOwner3 is not a public member of Azure. diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/AllMonikersEndsWithSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/AllMonikersEndsWithSourcePath new file mode 100644 index 00000000000..2292827b512 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/AllMonikersEndsWithSourcePath @@ -0,0 +1,6 @@ + +# AzureSdkOwners: @Azure/TestTeam3 +# ServiceLabel: %TestLabel3 +# ServiceOwners: @TestOwner3 +# PRLabel: %TestLabel0 +/sdk/someFakePath @TestOwner1 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/PRLabelAndSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/PRLabelAndSourcePath new file mode 100644 index 00000000000..a0a066c8521 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/PRLabelAndSourcePath @@ -0,0 +1,3 @@ + +# PRLabel: %TestLabel0 +/sdk/someFakePath @TestOwner1 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndMissingFolder b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndMissingFolder new file mode 100644 index 00000000000..36a11a5cc77 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndMissingFolder @@ -0,0 +1,3 @@ + +# ServiceLabel: %TestLabel1 +#// @Azure/TestTeam diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndSourcePath new file mode 100644 index 00000000000..6864799452e --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/ServiceLabelAndSourcePath @@ -0,0 +1,3 @@ +# ServiceLabel: %TestLabel2 +# The EOF is on the source path line. This comment is also part of the block but is ignored. +/sdk/someFakePath @TestOwner3 \ No newline at end of file diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/SingleSourcePathLine b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/SingleSourcePathLine new file mode 100644 index 00000000000..409796bf4db --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/FindBlockEnd/SingleSourcePathLine @@ -0,0 +1 @@ +/sdk/someFakePath @TestOwner1 \ No newline at end of file diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Matching/SubDirAndFallback b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Matching/SubDirAndFallback new file mode 100644 index 00000000000..299c89c2ee0 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/Matching/SubDirAndFallback @@ -0,0 +1,27 @@ +# CODEOWNERS with no errors, for match testing. +# The purpose of this file is for end-to-end GetMatchingCodeownersEntry with targeted subdirectory and fallback. +# The examples in here are real world. For example, many sdk service directories have azure resourcemanager +# library subdirectories and the following glob pattern, /sdk/**/azure-resourcemanager-*/, is used to get just +# these directories while leaving the base sdk service directory ownership alone. + +# Fallback +/** @TestTeam1 + +# PRLabel: %TestLabel1 +/sdk/ServiceDirectory1/ @TestOwner2 @TestTeam1 + +# PRLabel: %TestLabel2 +# ServiceLabel: %TestLabel3 +# AzureSdkOwners: @TestOwner4 +/sdk/ServiceDirectory2/ @TestOwner3 @TestTeam2 + +# This entry needs to be after the previous entries otherwise, by github rules, they'll take +# ownership. With this entry, anything in an azure-service1-* subdirectory of the previous two +# service directory entires will belong to these owners and have these labels +# PRLabel: %TestLabel4 +# ServiceLabel: %TestLabel2 +/sdk/**/azure-service1-*/ @TestTeam4 @TestOwner0 + +# This owner owns everything log related in the repository +# PRLabel: %TestLabel3 +/**/logs/* @TestOwner4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel new file mode 100644 index 00000000000..568753ae04d --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel @@ -0,0 +1,11 @@ +# Test scenario for AzureSdkOwners that need to be part of a block that contains a ServiceLabel entry. + +# AzureSdkOwners, whether owners are declared or not, needs to be part of a block that contains a ServiceLabel entry +# AzureSdkOwners: @TestOwner0 + +# Same scenario as above but with invalid/missing owners that shouldn't prevent the block formatting error from +# being reported +# AzureSdkOwners: @TestOwnerBad + +# AzureSdkOwners: + diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/DuplicateMonikers b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/DuplicateMonikers new file mode 100644 index 00000000000..88a68310db1 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/DuplicateMonikers @@ -0,0 +1,37 @@ +# Scenario data for duplicate moniker detection. + +# Duplicate PRLabel +# PRLabel: %TestLabel1 +# PRLabel: %TestLabelBad +/sdk/someFakePath @TestOwner2 + +# Duplicate AzureSdkOwners +# AzureSdkOwners: @TestUser0 +# AzureSdkOwners: @TestUserBad @TestOwner1 +# ServiceLabel: %TestLabel1 +# ServiceOwners: @TestOwner0 @TestOwner2 + +# Duplicate ServiceLabel ending in source path +# ServiceLabel: %TestLabel1 +# ServiceLabel: %TestLabelBad +/sdk/someFakePath @TestOwner2 + +# Duplicate ServiceLabel ending // +# ServiceLabel: %TestLabel1 +# ServiceLabel: %TestLabelBad +#// @TestOwner0 @TestOwnerBad + +# Duplicate ServiceLabel ending ServiceOwners +# ServiceLabel: %TestLabel1 +# ServiceLabel: %TestLabelBad +# ServiceOwners: @TestOwner0 @TestOwnerBad + +# Duplicate // +# ServiceLabel: %TestLabel1 +#// @TestOwner1 @TestOwnerBad +#// @TestOwner0 + +# Duplicate ServiceOwners +# ServiceLabel: %TestLabel1 +# ServiceOwners: @TestOwner0 @TestOwnerBad +# ServiceOwners: @TestOwner2 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersEndsInSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersEndsInSourcePath new file mode 100644 index 00000000000..f5cfad9685a --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersEndsInSourcePath @@ -0,0 +1,5 @@ +# All Monikers ends in source path/owner line +# AzureSdkOwners: @TestOwner2 +# PRLabel: %TestLabel1 +# ServiceLabel: TestLabel3 +/sdk/someFakePath @TestOwner0 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath new file mode 100644 index 00000000000..f6c1d862955 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath @@ -0,0 +1,19 @@ +# Test for error reporting. Monikers that need to be part of a block that ends in a source/path owner +# PRLabel: %TestLabel1 + +# Same scenario as above but with invalid/missing labels that shouldn't prevent the block +# formatting error from being reported +# PRLabel: TestLabelBad + +# PRLabel: + +# ServiceLabel, whether a label is declared or whether the label is valid or not, needs to be +# part of a block that ends in a source path/owner line, a ServiceOwner line or // +# ServiceLabel: TestLabel1 + +# Same scenario as above but with invalid/missing labels that shouldn't prevent the block +# formatting error from being reported +# ServiceLabel: %TestLabelBad + +# ServiceLabel: + diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/PRLabelAndSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/PRLabelAndSourcePath new file mode 100644 index 00000000000..2b822e526b0 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/PRLabelAndSourcePath @@ -0,0 +1,3 @@ +# A PRLabel block ending with a source path/owner line should have no errors +# PRLabel: %TestLabel1 +/sdk/someFakePath @TestOwner0 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndMissingPath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndMissingPath new file mode 100644 index 00000000000..4b4014ddb1c --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndMissingPath @@ -0,0 +1,3 @@ +# A ServiceLabel block ending with a // line should have no errors +# ServiceLabel: %TestLabel3 +#// @Azure/TestTeam3 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndServiceOwners b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndServiceOwners new file mode 100644 index 00000000000..fdb53a9e61a --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndServiceOwners @@ -0,0 +1,3 @@ +# A ServiceLabel block ending with a ServiceOwners should have no errors +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner2 @TestOwner4 @Azure/TestTeam4 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndSourcePath b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndSourcePath new file mode 100644 index 00000000000..3f1429d7d26 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelAndSourcePath @@ -0,0 +1,3 @@ +# A ServiceLabel block ending in a source path/owner line should have no errors +# ServiceLabel: %TestLabel4 +/sdk/someFakePath @TestOwner0 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers new file mode 100644 index 00000000000..2d27780fb8b --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers @@ -0,0 +1,16 @@ +# If a ServiceLabel ends with a source path/owner line and has ServiceOwners +# in the same block. +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner2 +/sdk/someFakePath @TestOwner0 + +# If a ServiceLabel ends with a source path/owner line and has the MissingPath +# in the same block. +# ServiceLabel: %TestLabel0 +#// @TestOwner0 +/sdk/someFakePath @TestOwner4 + +# If a ServiceLabel is part of a block that has both ServiceOwners and MissingPath +# ServiceLabel: %TestLabel4 +# ServiceOwners: @TestOwner2 +#// @TestOwner0 diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/SingleSourcePathLine b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/SingleSourcePathLine new file mode 100644 index 00000000000..cf72628261b --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/CodeownersTestFiles/VerifyBlock/SingleSourcePathLine @@ -0,0 +1,2 @@ +# A single source path/owner line is its own block and should have no errors. +/sdk/someFakePath @TestOwner0 \ No newline at end of file diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Mocks/DirectoryUtilsMock.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Mocks/DirectoryUtilsMock.cs new file mode 100644 index 00000000000..e9e9b867ccb --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Mocks/DirectoryUtilsMock.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Mocks +{ + /// + /// Mock DirectoryUtils class. This is necessary for NUnit tests which need to be able to run + /// without actually having a repository directory. + /// + public class DirectoryUtilsMock: DirectoryUtils + { + public DirectoryUtilsMock() + { + } + + // This is the only method that needs to be overridden to call nothing else and then return; + public override void VerifySourcePathEntry(string sourcePathEntry, List errorStrings) + { + return; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Parsing/CodeownersParserTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Parsing/CodeownersParserTests.cs new file mode 100644 index 00000000000..71d083f5833 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Parsing/CodeownersParserTests.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Parsing; +using Azure.Sdk.Tools.CodeownersUtils.Tests.Mocks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Parsing +{ + public class CodeownersParserTests + { + private OwnerDataUtils _ownerDataUtils; + private RepoLabelDataUtils _repoLabelDataUtils; + private DirectoryUtilsMock _directoryUtilsMock; + private CodeownersEntry _emptyEntry; + + [OneTimeSetUp] + // Initialize a DirectoryUtilsMock, OwnerDataUtils and RepoLabelDataUtils + public void InitTestData() + { + _directoryUtilsMock = new DirectoryUtilsMock(); + _ownerDataUtils = TestHelpers.SetupOwnerData(); + _repoLabelDataUtils = TestHelpers.SetupRepoLabelData(); + // None of the tests will work if the repo/label data doesn't exist. + // While the previous function call created the test repo/label data, + // this call just ensures no changes were made to the util that'll + // mess things up. + if (!_repoLabelDataUtils.RepoLabelDataExists()) + { + throw new ArgumentException($"Test repo/label data should have been created for {TestHelpers.TestRepositoryName} but was not."); + } + _emptyEntry = new CodeownersEntry(); + } + + /// + /// Test ParseCodeownersFile. Loads a test codeowners file and compares that against the expected output + /// deserialized from Json. The thing that's different about parsing vs verification is that any individual + /// lines, themselves, are not validted. The only thing that is validated is that the block doesn't have any + /// errors. Any blocks with errors are ignored and won't be in the returned list of Codeowners entries. + /// + /// The test CODEOWNERS file to parse + /// The json file with the expected Codeowners entries. + [TestCase("CodeownersTestFiles/EndToEnd/NoErrors", "CodeownersTestFiles/EndToEnd/NoErrorsExpectedEntries.json")] + // This is to ensure that block entries with errors are not parsed + [TestCase("CodeownersTestFiles/EndToEnd/WithBlockErrors", "CodeownersTestFiles/EndToEnd/WithBlockErrorsExpectedEntries.json")] + public void TestParseCodeownersFile(string codeownersFile, string jsonFileWithExpectedEntries) + { + List actualEntries = CodeownersParser.ParseCodeownersFile(codeownersFile); + + string expectedEntriesJson = FileHelpers.GetFileOrUrlContents(jsonFileWithExpectedEntries); + List expectedEntries = JsonSerializer.Deserialize>(expectedEntriesJson); + bool entiresAreEqual = TestHelpers.CodeownersEntryListsAreEqual(actualEntries, expectedEntries); + if (!entiresAreEqual) + { + Assert.Fail(TestHelpers.FormatCodeownersListDifferences(actualEntries, expectedEntries)); + } + } + + /// + /// Test the GetMatchingCodeownersEntry. Ultimately, this uses the DictoryUtils.PathExpressionMatchesTargetPath + /// to match the targetPath against the CodeownersEntry's PathExpression in reverse order and it's this piece + /// that's being tested. + /// + /// The test CODEOWNERS file to parse + /// The target path to find a match for + /// Whether or not a match is expected + /// If the match is expected, the PathExpression that is expected to match + // Targer path doesn't exist in the repository CODEOWNERS, should hit the global fallback + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/NotInCODEOWNERSFILE/foo.yml", true, "/**")] + // Default tests, these should straight up match the sdk/ServiceDirectory* entries + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory1/SomeSubDirectory1/SomeFile1.md", true, "/sdk/ServiceDirectory1/")] + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory2/SomeSubDirectory2/SomeFile2.md", true, "/sdk/ServiceDirectory2/")] + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory2/azure-service1-test/SomeFile3.md", true, "/sdk/**/azure-service1-*/")] + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory4/azure-service1-test2/SomeFile4.md", true, "/sdk/**/azure-service1-*/")] + // Entry in logs directory, regardless of where it is in the repository + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory1/azure-service1-test5/logs/LogFile.txt", true, "/**/logs/*")] + // Similar but not quite + [TestCase("CodeownersTestFiles/Matching/SubDirAndFallback", "sdk/ServiceDirectory1/azure-service1-test5/Notlogs/NotLogFile.txt", true, "/sdk/**/azure-service1-*/")] + public void TestGetMatchingCodeownersEntry(string codeownersFile, string targetPath, bool expectMatch, string expectedMatchPathExpression) + { + List parsedEntries = CodeownersParser.ParseCodeownersFile(codeownersFile); + CodeownersEntry matchedEntry = CodeownersParser.GetMatchingCodeownersEntry(targetPath, parsedEntries); + if (!expectMatch) + { + Assert.That(matchedEntry, Is.EqualTo(_emptyEntry), $"No match was expected but the following entry was returned.\n{matchedEntry}"); + } + else + { + Assert.That(matchedEntry.PathExpression, Is.EqualTo(expectedMatchPathExpression), $"Expected matching entry for targetPath, {targetPath}, to match {expectedMatchPathExpression} but {matchedEntry.PathExpression} was returned instead."); + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/TestHelpers.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/TestHelpers.cs new file mode 100644 index 00000000000..5b96e51c356 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/TestHelpers.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Caches; +using Azure.Sdk.Tools.CodeownersUtils.Parsing; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests +{ + public static class TestHelpers + { + public const string TestRepositoryName = @"Azure\azure-sdk-fake"; + // The partial string will have a number appended to the end of them + public const string TestLabelNamePartial = "TestLabel"; + public const string TestOwnerNamePartial = "TestOwner"; + public const string TestTeamNamePartial = "TestTeam"; + // This will control the max number of test users, test teams and test labels + private const int _maxItems = 5; + + /// + /// Create the team/user and user/org visibility data for the OwnerDataUtils to be used in testing. + /// The user/org visibility data will consist of 5 users, TestOwner0..TestOwner4, with only even users + /// being visible. + /// The team/user data will consist of 5 teams with the number of users in each team equal to the + /// team number, TestTeam0...TestTeam4.The users in each team will consist of the same users in + /// the user/org data. + /// + /// Populated OwnerDataUtils + public static OwnerDataUtils SetupOwnerData() + { + // OwnerDataUtils requires a TeamUserCache and a UserOrgVisibilityCache populated with their + // respective data. Live data really can't be used for this because it can change. + TeamUserCache teamUserCache = new TeamUserCache(null); + Dictionary> teamUserDict = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); + + // Create 5 teams + for (int i = 0;i < _maxItems; i++) + { + string team = $"{TestTeamNamePartial}{i}"; + List users = new List(); + // Teams will have a number of users equal to their team number with a + // max of 5. The user/org visibility will have data for each user. + // Note the stopping condition is j <= i, this is so team 0 has 1 user + // and team 4 has all five users + for (int j = 0; j <= i; j++) + { + string user = $"{TestOwnerNamePartial}{j}"; + users.Add(user); + } + teamUserDict.Add(team, users); + } + teamUserCache.TeamUserDict = teamUserDict; + + UserOrgVisibilityCache userOrgVisibilityCache = new UserOrgVisibilityCache(null); + Dictionary userOrgVisDict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + // Make every even user visible + for (int i = 0;i < 5;i++) + { + string user = $"{TestOwnerNamePartial}{i}"; + if (i % 2 == 0) + { + userOrgVisDict.Add(user, true); + } + else + { + userOrgVisDict.Add(user, false); + } + } + userOrgVisibilityCache.UserOrgVisibilityDict = userOrgVisDict; + return new OwnerDataUtils(teamUserCache, userOrgVisibilityCache); + } + + /// + /// Create the repo/label data for testing. + /// + /// Populated RepoLabelDataUtils + public static RepoLabelDataUtils SetupRepoLabelData() + { + Dictionary> repoLabelDict = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); + HashSet repoLabels = new HashSet(StringComparer.InvariantCultureIgnoreCase); + for (int i = 0;i < 5;i++) + { + string label = $"{TestLabelNamePartial}{i}"; + repoLabels.Add(label); + } + // Last but not least, add the Service Attention Label + repoLabels.Add(LabelConstants.ServiceAttention); + repoLabelDict.Add(TestRepositoryName, repoLabels); + RepoLabelCache repoLabelCache = new RepoLabelCache(null); + repoLabelCache.RepoLabelDict = repoLabelDict; + return new RepoLabelDataUtils(repoLabelCache, TestRepositoryName); + } + + /// + /// Given the actual and expected lists of errors, labels or owners, verify they both contain the same items. + /// + /// The actual list of errors, labels or owners + /// The expected list of errors, labels or owners + /// True if the lists are both equal, false otherwise + public static bool StringListsAreEqual(List actuaList, List expectedList) + { + return actuaList.SequenceEqual(expectedList); + } + + /// + /// Given two CodeownersEntry lists, verify they're equal. + /// + /// The actual, or parsed, list of Codeowners entries. + /// The expected list of Codeowners entries + /// True if they're equal, false otherwise + public static bool CodeownersEntryListsAreEqual(List actualCodeownerEntries, + List expectedCodeownerEntries) + { + // SequenceEqual determines whether two sequences are equal by comparing the length and + // then by comparing the elements by using the default equality comparer for their type. + // This has the side benefit of testing CodeownersEntry's Equals operator which compares + // the PathExpression and every moniker list of labels and owners for equality + return actualCodeownerEntries.SequenceEqual(expectedCodeownerEntries); + } + + /// + /// Given two list of Codeowners entries, format a string to be printed out with a test failure. + /// + /// The actual, or parsed, list of Codeowners entries. + /// The expected list of Codeowners entries + /// A formatted string containing the differences. + public static string FormatCodeownersListDifferences(List actualCodeownerEntries, + List expectedCodeownerEntries) + { + string diffString = ""; + // Get the items that are only in the actualList + List diffActual = actualCodeownerEntries.Except(expectedCodeownerEntries).ToList(); + // Get the items that are only in the expectedList + List diffExpected = expectedCodeownerEntries.Except(actualCodeownerEntries).ToList(); + + // Entries that do compare the same will not be in either list. The combination of the two lists are + // the diffs which can consiste of both completely missing entries and entries that don't compare the same. + if (diffActual.Count > 0 && diffExpected.Count > 0) + { + diffString = "The list of parsed extries and expected entries had differences in both lists.\n"; + diffString += "Parsed entries not in the expected entries:\n\n"; + diffString += string.Join(Environment.NewLine, diffActual.Select(d => d.ToString())); + diffString += "\n\nExpected entries not in the parsed entries:\n"; + diffString += string.Join(Environment.NewLine, diffExpected.Select(d => d.ToString())); + } + // This is the case where the actual contained more entries than the expected + else if (diffActual.Count > 0) + { + diffString = "The list of parsed entries contains more items than expected. The additional items are as follows:\n"; + diffString += string.Join(Environment.NewLine, diffActual.Select(d => d.ToString())); + } + // this is the case where only the expected contained more entries than the actual + else + { + diffString = "The list of expected entries contains more items than parsed. The additional items are as follows:\n"; + diffString += string.Join(Environment.NewLine, diffExpected.Select(d => d.ToString())); + } + + return diffString; + } + + /// + /// Given a list of BaseError, create a string with embedded newlines, that can be used in reporting + /// in test failures. + /// + /// List<BaseError> that need to be formatted. + /// string, formatted list of errors + public static string FormatErrorMessageFromErrorList(List errors) + { + string errorString = ""; + foreach (BaseError error in errors) + { + errorString += error + Environment.NewLine; + } + return errorString; + } + + /// + /// This function is used to help generate the json strings that will be deserialized for the parsing tests. + /// + /// List<CodeownersEntry> to be serialized + public static void TempGetSerializedString(List codeownersEntries) + { + JsonSerializerOptions jsonSerializerOptions = new() + { + WriteIndented = true + }; + string jsonString = JsonSerializer.Serialize(codeownersEntries, jsonSerializerOptions); + Console.WriteLine(jsonString); + Console.WriteLine("something to break on"); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/BaselineUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/BaselineUtilsTests.cs new file mode 100644 index 00000000000..d886b6bf6ea --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/BaselineUtilsTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Tests.Mocks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + public class BaselineUtilsTests + { + private OwnerDataUtils _ownerDataUtils; + private RepoLabelDataUtils _repoLabelDataUtils; + private DirectoryUtilsMock _directoryUtilsMock; + private List _tempFiles = new List(); + + [OneTimeSetUp] + // Initialize a DirectoryUtilsMock, OwnerDataUtils and RepoLabelDataUtils + public void InitTestData() + { + _directoryUtilsMock = new DirectoryUtilsMock(); + _ownerDataUtils = TestHelpers.SetupOwnerData(); + _repoLabelDataUtils = TestHelpers.SetupRepoLabelData(); + // None of the tests will work if the repo/label data doesn't exist. + // While the previous function call created the test repo/label data, + // this call just ensures no changes were made to the util that'll + // mess things up. + if (!_repoLabelDataUtils.RepoLabelDataExists()) + { + throw new ArgumentException($"Test repo/label data should have been created for {TestHelpers.TestRepositoryName} but was not."); + } + } + + [OneTimeTearDown] + public void CleanupTestData() + { + int maxTries = 5; + // Cleanup any temporary files created by the tests + if (_tempFiles.Count > 0) + { + foreach(string tempFile in _tempFiles) + { + int tryNumber = 0; + while (tryNumber < maxTries) + { + try + { + File.Delete(tempFile); + break; + } + catch (Exception ex) + { + Console.WriteLine($"Exception trying to delete {tempFile}. Ex={ex}"); + // sleep 100ms and try again + System.Threading.Thread.Sleep(100); + } + tryNumber++; + } + // Report that the test wasn't able to delete the file after successive tries + if (tryNumber == maxTries) + { + Console.WriteLine($"Unable to delete {tempFile} after {maxTries}, see above for exception details."); + } + } + } + } + + /// + /// Test the baseline generation. Only line errors should be in the baseline, not block errors. + /// + /// The CODEOWNERS file to generate the baseline for. + /// The baseline file that is known to be correct for the CODEOWNERS file. + [TestCase("CodeownersTestFiles/Baseline/WithBlockErrors", "CodeownersTestFiles/Baseline/WithBlockErrors_FullBaseline.txt", 4)] + [TestCase("CodeownersTestFiles/Baseline/NoErrors", "CodeownersTestFiles/Baseline/NoErrors_EmptyBaseline.txt", 0)] + public void TestBaselineGenerationAndFiltering(string testCodeownerFile, + string expectedBaselineFile, + int expectedNumberOfBlockErrors) + { + List actualErrors = CodeownersLinter.LintCodeownersFile(_directoryUtilsMock, + _ownerDataUtils, + _repoLabelDataUtils, + testCodeownerFile); + + // Load the expected baseline file and, for sanity's sake, ensure that all errors are filtered + BaselineUtils expectedBaselineUtils = new BaselineUtils(expectedBaselineFile); + var filteredWithExpected = expectedBaselineUtils.FilterErrorsUsingBaseline(actualErrors); + + // This piece is for sanity, to verify that the the filter only filters out SingleLineErrors and leaves + // the BlockFormattingErrors. + if (expectedNumberOfBlockErrors > 0) + { + if (filteredWithExpected.Count != expectedNumberOfBlockErrors) + { + string errorString = TestHelpers.FormatErrorMessageFromErrorList(filteredWithExpected); + Assert.Fail($"The expected baseline file {expectedBaselineFile} for test CODEOWNERS file {testCodeownerFile} should have had {expectedNumberOfBlockErrors} block errors but had {filteredWithExpected.Count}. Unexpected baseline errors\n{errorString}"); + } + } + else + { + if (filteredWithExpected.Count > 0) + { + string errorString = TestHelpers.FormatErrorMessageFromErrorList(filteredWithExpected); + Assert.Fail($"The expected baseline file {expectedBaselineFile} for test CODEOWNERS file {testCodeownerFile} should have filtered out all the errors but filtering return {filteredWithExpected.Count} errors. Unexpected baseline errors\n{errorString}"); + } + } + + // Since the filtering has been verified above, regenerate the filter file and ensure that the generation produces + // the same results. AKA only the SingleLineErrors are filtered out. + string baselineTemp = GenerateTempFile(); + BaselineUtils generatedBaselineUtils = new BaselineUtils(baselineTemp); + generatedBaselineUtils.GenerateBaseline(actualErrors); + var filteredWithGenerated = generatedBaselineUtils.FilterErrorsUsingBaseline(actualErrors); + + if (expectedNumberOfBlockErrors > 0) + { + if (filteredWithGenerated.Count != expectedNumberOfBlockErrors) + { + string errorString = TestHelpers.FormatErrorMessageFromErrorList(filteredWithGenerated); + Assert.Fail($"The generated baseline file {expectedBaselineFile} for test CODEOWNERS file {testCodeownerFile} should have had {expectedNumberOfBlockErrors} block errors but had {filteredWithGenerated.Count}. Unexpected baseline errors\n{errorString}"); + } + } + else + { + if (filteredWithGenerated.Count > 0) + { + string errorString = TestHelpers.FormatErrorMessageFromErrorList(filteredWithGenerated); + Assert.Fail($"The generated baseline file {expectedBaselineFile} for test CODEOWNERS file {testCodeownerFile} should have filtered out all the errors but filtering return {filteredWithGenerated.Count} errors. Unexpected baseline errors\n{errorString}"); + } + } + } + + /// + /// Generate a temporary file to be used for testing and save the filename + /// to be cleaned up in a function called in a OneTimeTearDown + /// + /// string which is the full path to the temp file + private string GenerateTempFile() + { + string tempFile = Path.GetTempFileName(); + _tempFiles.Add(tempFile); + return tempFile; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/DirectoryUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/DirectoryUtilsTests.cs new file mode 100644 index 00000000000..b9dc412cc1c --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/DirectoryUtilsTests.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + [TestFixture] + [Parallelizable(ParallelScope.Children)] + /// + /// DirectoryUtils tests will not test every function in the DirectoryUtils. The reasons for this are + /// partially because it requires a repository with a CODEOWNERS to run. Further, IsValidGlobPatternForRepo + /// is effectively a wrapper around FileSystemGlobbing's matcher and IsValidRepositoryPath is a wrapper + /// that just checks Directory.Exists || File.Exists on the passed in source path. + /// The only function that really needs to be tested is IsValidCodeownersPathExpression which is checking + /// the source path for invalid CODEOWNERS patterns. + /// + public class DirectoryUtilsTests + { + /// + /// Verify that IsValidCodeownersPathExpression returns true and has no errors with valid glob patterns + /// + /// Valid glob pattern + [Category("Utils")] + [Category("Directory")] + // These are valid glob patterns taken from various azure-sdk CODEOWNERS files + [TestCase("/sdk/**/azure-resourcemanager-*/")] + [TestCase("/**/README.md")] + [TestCase("/*.md")] + [TestCase("/SomeDirectory/*")] + [TestCase("/**")] + public void TestValidCodeownersGlobPatterns(string glob) + { + // For the purposes of testing this function, the repoRoot is not necessary. + List errorStrings = new List(); + bool isValid = DirectoryUtils.IsValidCodeownersPathExpression(glob, errorStrings); + Assert.IsTrue(isValid, $"IsValidCodeownersPathExpression for '{glob}' should have returned true but did not."); + Assert.That(errorStrings.Count, Is.EqualTo(0), $"IsValidCodeownersPathExpression for '{glob}' should not have returned any error strings for a valid pattern but returned {errorStrings.Count}."); + } + + /// + /// Verify that IsValidCodeownersPathExpression returns false with the expected error string for + /// invalid glob patterns. Invalid glob patterns for CODEOWNERS are defined in the GitHub docs + /// https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax + /// While a line can contain multiple errors, the testcases are each setup to target an verify a specific error. + /// + /// The glob pattern to test. Each pattern returns a specific error. + /// The expected error message. + /// Whether or not the error message is partial and needs the glob prepended to match. + [Category("Utils")] + [Category("Directory")] + // Contains invalid characters. Note: everything starts with a slash otherwise it'll give the "must start with a /" + // error on top of the expected error. + [TestCase("/\\#*\\#", ErrorMessageConstants.ContainsEscapedPoundPartial)] + [TestCase("/!.vscode/cspell.json", ErrorMessageConstants.ContainsNegationPartial)] + [TestCase("/*.[Rr]e[Ss]harper", ErrorMessageConstants.ContainsRangePartial)] + [TestCase("/s?b", ErrorMessageConstants.ContainsQuestionMarkPartial)] + // contains invalid sequences + [TestCase("/", ErrorMessageConstants.PathIsSingleSlash, false)] + [TestCase("/**/", ErrorMessageConstants.PathIsSingleSlashTwoAsterisksSingleSlash, false)] + [TestCase("/sdk/foo*", ErrorMessageConstants.GlobCannotEndInWildCardPartial)] + [TestCase("/sdk/**/", ErrorMessageConstants.GlobCannotEndWithSingleSlashTwoAsterisksSingleSlashPartial)] + [TestCase("/sdk/**", ErrorMessageConstants.GlobCannotEndWithSingleSlashTwoAsterisksPartial)] + [TestCase("sdk/whatever/", ErrorMessageConstants.MustStartWithASlashPartial)] + public void TestInvalidCodeownersGlobPatterns(string glob, string expectedError, bool isPartial=true) + { + string expectedErrorMessage = expectedError; + if (isPartial) + { + expectedErrorMessage = $"{glob}{expectedError}"; + } + + // For the purposes of testing this function, the repoRoot is not necessary. + List errorStrings = new List(); + bool isValid = DirectoryUtils.IsValidCodeownersPathExpression(glob, errorStrings); + Assert.IsFalse(isValid, $"IsValidCodeownersPathExpression for '{glob}' should have returned false but did not."); + Assert.That(errorStrings.Count, Is.EqualTo(1), $"IsValidCodeownersPathExpression for '{glob}' should have returned a single error string but returned {errorStrings.Count} error strings."); + Assert.That(errorStrings[0], Is.EqualTo(expectedErrorMessage), $"IsValidCodeownersPathExpression for '{glob}' should have returned '{expectedErrorMessage}' but returned '{errorStrings[0]}'"); + } + + /// + /// Test PathExpressionMatchesTargetPath. + /// + /// + /// + /// + [TestCase("/**", "a", true)] + [TestCase("/**", "A", true)] + [TestCase("/**", "/a", true)] + [TestCase("/**", "a/", true)] + [TestCase("/**", "/a/", true)] + [TestCase("/**", "/a/b", true)] + [TestCase("/**", "/a/b/", true)] + [TestCase("/**", "/a/b/c", true)] + [TestCase("/**", "[", true)] + [TestCase("/**", "]", true)] + [TestCase("/", "a", false)] + [TestCase("/", "A", false)] + [TestCase("/", "/a", false)] + [TestCase("/", "a/", false)] + [TestCase("/", "/a/", false)] + [TestCase("/", "/a/b", false)] + [TestCase("/a", "a", true)] + [TestCase("/a", "A", false)] + [TestCase("/a", "/a", true)] + [TestCase("/a", "a/", false)] + [TestCase("/a", "/a/", false)] + [TestCase("/a", "/a/b", false)] + [TestCase("/a", "/a/b/", false)] + [TestCase("/a", "/a\\ b", false)] + [TestCase("/a", "/x/a/b", false)] + [TestCase("a", "a", false)] + [TestCase("a", "ab", false)] + [TestCase("a", "ab/", false)] + [TestCase("a", "/ab/", false)] + [TestCase("a", "A", false)] + [TestCase("a", "/a", false)] + [TestCase("a", "a/", false)] + [TestCase("a", "/a/", false)] + [TestCase("a", "/a/b", false)] + [TestCase("a", "/a/b/", false)] + [TestCase("a", "/x/a/b", false)] + [TestCase("/a/", "a", false)] + [TestCase("/a/", "/a", false)] + [TestCase("/a/", "a/", true)] + [TestCase("/a/", "/a/", true)] + [TestCase("/a/", "/a/b", true)] + [TestCase("/a/", "/a/a\\ b/", true)] + [TestCase("/a/", "/a/b/", true)] + [TestCase("/a/", "/A/b/", false)] + [TestCase("/a/", "/x/a/b", false)] + [TestCase("/a/b/", "/a", false)] + [TestCase("/a/b/", "/a/", false)] + [TestCase("/a/b/", "/a/b", false)] + [TestCase("/a/b/", "/a/b/", true)] + [TestCase("/a/b/", "/a/b/c", true)] + [TestCase("/a/b/", "/a/b/c/", true)] + [TestCase("/a/b/", "/a/b/c/d", true)] + [TestCase("/a/b", "/a", false)] + [TestCase("/a/b", "/a/", false)] + [TestCase("/a/b", "/a/b", true)] + [TestCase("/a/b", "/a/b/", false)] + [TestCase("/a/b", "/a/bc", false)] + [TestCase("/a/b", "/a/bc/", false)] + [TestCase("/a/b", "/a/b/c", false)] + [TestCase("/a/b", "/a/b/c/", false)] + [TestCase("/a/b", "/a/b/c/d", false)] + [TestCase("/!a", "!a", false)] + [TestCase("/!a", "b", false)] + [TestCase("/a[b", "a[b", false)] + [TestCase("/a]b", "a]b", false)] + [TestCase("/a?b", "a?b", false)] + [TestCase("/a?b", "axb", false)] + [TestCase("/a", "*", false)] + [TestCase("/*", "*", false)] + [TestCase("/*", "a", true)] + [TestCase("/*", "a/", false)] + [TestCase("/*", "/a", true)] + [TestCase("/*", "/a/", false)] + [TestCase("/*", "a/b", false)] + [TestCase("/*", "/a/b", false)] + [TestCase("/*", "[", true)] + [TestCase("/*", "]", true)] + [TestCase("/*", "!", true)] + [TestCase("/**", "!", true)] + [TestCase("/a*", "a/x", false)] + [TestCase("/a*", "a/x/d", false)] + [TestCase("/a*.md", "ab.md", true)] + [TestCase("/a*", "ab/x", false)] + [TestCase("/a*", "ab/x/d", false)] + [TestCase("/a/**", "a", false)] + [TestCase("/*/**", "a", false)] + [TestCase("/*/**", "a/", false)] + [TestCase("/*/**", "a/b", false)] + [TestCase("/*/", "a", false)] + [TestCase("/*/", "a/", true)] + [TestCase("/*/b", "a/b", true)] + [TestCase("/**/a", "a", true)] + [TestCase("/**/a", "x/ba", false)] + [TestCase("/a/*", "a", false)] + [TestCase("/a/*", "a/", true)] + [TestCase("/a/*", "a/b", true)] + [TestCase("/a/*", "a/b/", false)] + [TestCase("/a/*", "a/b/c", false)] + [TestCase("/a/*/", "a", false)] + [TestCase("/a/*/", "a/", false)] + [TestCase("/a/*/", "a/b", false)] + [TestCase("/a/*/", "a/b/", true)] + [TestCase("/a/*/", "a/b/c", true)] + [TestCase("/a/**", "a", false)] + [TestCase("/a/**", "a/", false)] + [TestCase("/a/**", "a/b", false)] + [TestCase("/a/**", "a/b/", false)] + [TestCase("/a/**", "a/b/c", false)] + [TestCase("/a/**/", "a", false)] + [TestCase("/a/**/", "a/", false)] + [TestCase("/a/**/", "a/b", false)] + [TestCase("/a/**/", "a/b/", false)] + [TestCase("/a/**/", "a/b/c", false)] + [TestCase("/**/a/", "a", false)] + [TestCase("/**/a/", "a/", true)] + [TestCase("/**/a/", "a/b", true)] + [TestCase("/**/b/", "a/b", false)] + [TestCase("/**/b/", "a/b/", true)] + [TestCase("/**/b/", "a/c/", false)] + [TestCase("/a/*/b/", "a/b/", false)] + [TestCase("/a/*/b/", "a/x/b/", true)] + [TestCase("/a/*/b/", "a/x/b/c", true)] + [TestCase("/a/*/b/", "a/x/c", false)] + [TestCase("/a/*/b/", "a/x/y/b", false)] + [TestCase("/a**b/", "a/x/y/b", false)] + [TestCase("/a/**/b/", "a/b", false)] + [TestCase("/a/**/b/", "a/b/", true)] + [TestCase("/a/**/b/", "a/x/b/", true)] + [TestCase("/a/**/b/", "a/x/y/b/", true)] + [TestCase("/a/**/b/", "a/x/y/c", false)] + [TestCase("/a/**/b/", "a-b/", false)] + [TestCase("a/*/*", "a/b", false)] + [TestCase("/a/*/*/d", "a/b/c/d", true)] + [TestCase("/a/*/*/d", "a/b/x/c/d", false)] + [TestCase("/a/**/*/d", "a/b/x/c/d", true)] + [TestCase("*/*/b", "a/b", false)] + [TestCase("/a*/", "abc/", true)] + [TestCase("/a*/", "ab/c/", true)] + [TestCase("/*b*/", "axbyc/", true)] + [TestCase("/*c/", "abc/", true)] + [TestCase("/*c/", "a/abc/", false)] + [TestCase("/a*c/", "axbyc/", true)] + [TestCase("/a*c/", "axb/yc/", false)] + [TestCase("/**/*x*/", "a/b/cxy/d", true)] + [TestCase("/a/*.md", "a/x.md", true)] + [TestCase("/*/*/*.md", "a/b/x.md", true)] + [TestCase("/**/*.md", "a/b.md/x.md", true)] + [TestCase("**/*.md", "a/b.md/x.md", false)] + [TestCase("/*.md", "a/md", false)] + [TestCase("/a.*", "a.b", false)] + [TestCase("/a.*", "x/a.b/", false)] + [TestCase("/a.*/", "a.b", false)] + [TestCase("/a.*/", "a.b/", true)] + [TestCase("/**/*x*/AB/*/CD", "a/b/cxy/AB/fff/CD", true)] + [TestCase("/**/*x*/AB/*/CD", "a/b/cxy/AB/ff/ff/CD", false)] + [TestCase("/**/*x*/AB/**/CD/*", "a/b/cxy/AB/ff/ff/CD", false)] + [TestCase("/**/*x*/AB/**/CD/*", "a/b/cxy/AB/ff/ff/CD/", true)] + [TestCase("/**/*x*/AB/**/CD/*", "a/b/cxy/AB/[]/!!/CD/h", true)] + public void TestPathExpressionMatchesTargetPath(string pathExpression, string targetPath, bool expectMatch) + { + bool hasMatch = DirectoryUtils.PathExpressionMatchesTargetPath(pathExpression, targetPath); + Assert.That(hasMatch, Is.EqualTo(expectMatch), $"The pathExpression, {pathExpression}, for targetPath, {targetPath}, should have returned {expectMatch} and did not."); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/MonikerUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/MonikerUtilsTests.cs new file mode 100644 index 00000000000..4540a9af40d --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/MonikerUtilsTests.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + [TestFixture] + [Parallelizable(ParallelScope.Children)] + // All of these tests can run in parallel + public class MonikerUtilsTests + { + [Category("Utils")] + [Category("Moniker")] + // These should be using the SeparatorConstants.Colon but the interpolated strings doesn't seem to want to accept this + [TestCase(MonikerConstants.AzureSdkOwners, $"# {MonikerConstants.AzureSdkOwners}: @fakeOwner1 @fakeOwner2")] + [TestCase(MonikerConstants.MissingFolder, $"#{MonikerConstants.MissingFolder} @fakeOwner1 @fakeOwner2")] + [TestCase(MonikerConstants.PRLabel, $"# {MonikerConstants.PRLabel}: %Fake Label")] + [TestCase(MonikerConstants.ServiceLabel, $"# {MonikerConstants.ServiceLabel}: %Fake Label")] + [TestCase(MonikerConstants.ServiceOwners, $"# {MonikerConstants.ServiceOwners}:")] + // Moniker only lines, without their owners or labels, still need to be positively identified + [TestCase(MonikerConstants.AzureSdkOwners, $"# {MonikerConstants.AzureSdkOwners}:")] + [TestCase(MonikerConstants.MissingFolder, $"#{MonikerConstants.MissingFolder}")] + [TestCase(MonikerConstants.PRLabel, $"# {MonikerConstants.PRLabel}:")] + [TestCase(MonikerConstants.ServiceLabel, $"# {MonikerConstants.ServiceLabel}:")] + [TestCase(MonikerConstants.ServiceOwners, $"# {MonikerConstants.ServiceOwners}:")] + public void TestMonikerParsingForMonikerLines(string moniker, string line) + { + // The MonikerUtils has 2 methods to test for Moniker parsing + // 1. ParseMonikerFromLine - returns the moniker if one is found on the line + // 2. IsMonikerLine - returns true if the line is a moniker line + bool isMonikerLine = MonikerUtils.IsMonikerLine(line); + Assert.IsTrue(isMonikerLine, $"IsMonikerLine for '{line}' contains '{moniker}' and should have returned true."); + string parsedMoniker = MonikerUtils.ParseMonikerFromLine(line); + Assert.That(parsedMoniker, Is.EqualTo(moniker), $"ParseMonikerFromLine for '{line}' should have returned '{moniker}' but returned '{parsedMoniker}'"); + } + + [Category("Utils")] + [Category("Moniker")] + [TestCase("# just a comment line")] + // Whitespace line with spaces and tabs + [TestCase(" \t")] + [TestCase("")] + public void TestMonikerParsingForNonMonikerLines(string sourceLine) + { + bool isMonikerLine = MonikerUtils.IsMonikerLine(sourceLine); + Assert.IsFalse(isMonikerLine, $"IsMonikerLine for '{sourceLine}' does not contain a moniker and should have returned false."); + string parsedMoniker = MonikerUtils.ParseMonikerFromLine(sourceLine); + Assert.That(parsedMoniker, Is.EqualTo(null), $"ParseMonikerFromLine for '{sourceLine}' should have returned 'null' but returned '{parsedMoniker}'"); + } + + [Category("Utils")] + [Category("Moniker")] + // Source line with owners + [TestCase("/fakePath1/fakePath2 @fakeOwner1 @fakeOwner2")] + // Source line with no owners + [TestCase("/fakePath1/fakePath2")] + public void TestIsMonikerOrSourceLineForSourceLines(string sourceLine) + { + bool isMonikerLine = MonikerUtils.IsMonikerLine(sourceLine); + Assert.IsFalse(isMonikerLine, $"IsMonikerLine for '{sourceLine}' should have returned false for a source line."); + string parsedMoniker = MonikerUtils.ParseMonikerFromLine(sourceLine); + Assert.That(parsedMoniker, Is.EqualTo(null), $"ParseMonikerFromLine for '{sourceLine}' should have returned 'null' for a source line but returned '{parsedMoniker}'"); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/OwnerDataUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/OwnerDataUtilsTests.cs new file mode 100644 index 00000000000..c94d3e1d498 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/OwnerDataUtilsTests.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Caches; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + [TestFixture] + [Parallelizable(ParallelScope.Children)] + /// + /// OwnerDataUtilsTests requires an OwnerDataUtils with both TeamUserCache and UserOrgVisibilityCache pre-populated + /// with test data. + /// + public class OwnerDataUtilsTests + { + private OwnerDataUtils _ownerDataUtils; + + [OneTimeSetUp] + public void InitOwnerData() + { + _ownerDataUtils = TestHelpers.SetupOwnerData(); + } + + /// + /// This is testing the OwnerDataUtil's IsWriteOwner function. In GitHub, the + /// ownerName is case insensitive but case preserving which means that an owner's + /// login could be OwnerName but CODEOWNERS file can have ownerName or ownername + /// or OwnerName and they're all the same owner. This means that the unlying struture + /// holding the owners, Dictionary<string, bool%gt, needs to be created with + /// a case insensitive comparison. + /// + /// The owner to check + /// True if the owner being tested should be found in the write owners. + [Category("Utils")] + [Category("Owner")] + [TestCase($"{TestHelpers.TestOwnerNamePartial}0", true)] + [TestCase($"{TestHelpers.TestOwnerNamePartial}1", true)] + [TestCase($"{TestHelpers.TestOwnerNamePartial}5", false)] + public void TestIsWriteOwner(string owner, bool expectedResult) + { + // Test owner lookup with preserved case owner + bool isWriteOwner = _ownerDataUtils.IsWriteOwner(owner); + Assert.That(isWriteOwner, Is.EqualTo(expectedResult), $"IsWriteOwner for {owner} should have returned {expectedResult} and did not"); + + // Test owner lookup with uppercase owner + string ownerUpper = owner.ToUpper(); + isWriteOwner = _ownerDataUtils.IsWriteOwner(ownerUpper); + Assert.That(isWriteOwner, Is.EqualTo(expectedResult), $"IsWriteOwner for {ownerUpper} should have returned {expectedResult} and did not"); + + // Test owner lookup with lowercase owner + string ownerLower = owner.ToLower(); + isWriteOwner = _ownerDataUtils.IsWriteOwner(ownerLower); + Assert.That(isWriteOwner, Is.EqualTo(expectedResult), $"IsWriteOwner for {ownerLower} should have returned {expectedResult} and did not"); + } + + /// + /// This is testing the OwnerDataUtil's IsWriteTeam function. In GitHub, the + /// team name is case insensitive but case preserving which means that an team + /// could be TeamName but CODEOWNERS file can have teamName or teamname + /// or TeamName and they're all the same team. This means that the unlying struture + /// holding the team/user data, Dictionary<string, List<string>, needs to + /// be created with a case insensitive comparison. + /// + /// The team name to check + /// True if the team being tested should be found in the team/user dictionary. + [Category("Utils")] + [Category("Owner")] + [TestCase($"{TestHelpers.TestTeamNamePartial}2", true)] + [TestCase($"{TestHelpers.TestTeamNamePartial}4", true)] + [TestCase($"{TestHelpers.TestTeamNamePartial}6", false)] + public void TestIsWriteTeam(string team, bool expectedResult) + { + // Test team lookup with preserved case team + bool isWriteTeam = _ownerDataUtils.IsWriteTeam(team); + Assert.That(isWriteTeam, Is.EqualTo(expectedResult), $"IsWriteTeam for {team} should have returned {expectedResult} and did not"); + + // Test team lookup with uppercase team + string teamUpper = team.ToUpper(); + isWriteTeam = _ownerDataUtils.IsWriteTeam(teamUpper); + Assert.That(isWriteTeam, Is.EqualTo(expectedResult), $"IsWriteTeam for {teamUpper} should have returned {expectedResult} and did not"); + + // Test team lookup with lowercase team + string teamLower = team.ToLower(); + isWriteTeam = _ownerDataUtils.IsWriteTeam(teamLower); + Assert.That(isWriteTeam, Is.EqualTo(expectedResult), $"IsWriteTeam for {teamLower} should have returned {expectedResult} and did not"); + } + + /// + /// This is testing the OwnerDataUtil's IsPublicAzureMember function. In GitHub, the + /// ownerName is case insensitive but case preserving which means that an owner's + /// login could be OwnerName but CODEOWNERS file can have ownerName or ownername + /// or OwnerName and they're all the same owner. This means that the unlying struture + /// holding the owners, Dictionary<string, bool%gt, needs to be created with + /// a case insensitive comparison. If an owner doesn't exist, the call returns false. + /// + /// The owner to check + /// True if the owner should be public. + [Category("Utils")] + [Category("Owner")] + // Even users are public, odd users are not. + [TestCase($"{TestHelpers.TestOwnerNamePartial}0", true)] + [TestCase($"{TestHelpers.TestOwnerNamePartial}3", false)] + // This user does not exist + [TestCase($"{TestHelpers.TestOwnerNamePartial}5", false)] + public void TestIsPublicAzureMember(string owner, bool expectedResult) + { + // Test owner lookup with preserved case owner + bool isPublicAzureMember = _ownerDataUtils.IsPublicAzureMember(owner); + Assert.That(isPublicAzureMember, Is.EqualTo(expectedResult), $"isPublicAzureMember for {owner} should have returned {expectedResult} and did not"); + + // Test owner lookup with uppercase owner + string ownerUpper = owner.ToUpper(); + isPublicAzureMember = _ownerDataUtils.IsPublicAzureMember(ownerUpper); + Assert.That(isPublicAzureMember, Is.EqualTo(expectedResult), $"isPublicAzureMember for {ownerUpper} should have returned {expectedResult} and did not"); + + // Test owner lookup with lowercase owner + string ownerLower = owner.ToLower(); + isPublicAzureMember = _ownerDataUtils.IsPublicAzureMember(ownerLower); + Assert.That(isPublicAzureMember, Is.EqualTo(expectedResult), $"isPublicAzureMember for {ownerLower} should have returned {expectedResult} and did not"); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/ParsingUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/ParsingUtilsTests.cs new file mode 100644 index 00000000000..0a7474b4fff --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/ParsingUtilsTests.cs @@ -0,0 +1,281 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + public class ParsingUtilsTests + { + private OwnerDataUtils _ownerDataUtils; + + [OneTimeSetUp] + public void InitRepoLabelData() + { + _ownerDataUtils = TestHelpers.SetupOwnerData(); + } + + /// + /// Test the ParsingUtils.ParseLabelsFromLine parser. The Moniker should be irrelevant since the parser should + /// be able to parse the owners regardless. This needs to be able to check the old sytax and the new + /// syntax. The old syntax required the percent sign before the labels because ServiceLabel required + /// the service label AND the Service Attention label because a tool was used to generate the FabricBot + /// JSon for a rule from these. The new syntax just requires Moniker: Label, where everything after the + /// colon is the label. + /// + /// CODEOWNERS line to parse + /// Expected list of labels to be parsed. Note, List<string> would be ideal here but NUnit requires a constant expression and won't allow one to be constructed. + [Category("Labels")] + [Category("Parsing")] + // Ensure that parsing doesn't fail if the line doesn't have any labels + // For the other test, ones with labels, test cases and ensure that spaces vs tabs don't matter for parsing. + [TestCase($"# {MonikerConstants.ServiceLabel}:")] + // The % sign should be {SeparatorConstants.Label} but NUnit isn't allowing the + // character constant in the string declaration + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}0", + $"{TestHelpers.TestLabelNamePartial}0")] + [TestCase($"# {MonikerConstants.PRLabel}:\t%{TestHelpers.TestLabelNamePartial}4", + $"{TestHelpers.TestLabelNamePartial}4")] + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}1\t%{TestHelpers.TestLabelNamePartial}2", + $"{TestHelpers.TestLabelNamePartial}1", + $"{TestHelpers.TestLabelNamePartial}2")] + [TestCase($"# {MonikerConstants.ServiceLabel}:\t%{TestHelpers.TestLabelNamePartial}3 %{TestHelpers.TestLabelNamePartial}4", + $"{TestHelpers.TestLabelNamePartial}3", + $"{TestHelpers.TestLabelNamePartial}4")] + // The new syntax doesn't have {SeparatorConstants.Label} before the label, everything after + // the : + [TestCase($"# {MonikerConstants.PRLabel}: {TestHelpers.TestLabelNamePartial}1", + $"{TestHelpers.TestLabelNamePartial}1")] + [TestCase($"# {MonikerConstants.ServiceLabel}: \t{TestHelpers.TestLabelNamePartial}2", + $"{TestHelpers.TestLabelNamePartial}2")] + public void TestParseLabelsFromLine(string line, params string[] expectedLabels) + { + // Convert the array to List + var expectedLabelsList = expectedLabels.ToList(); + var parsedLabelsList = ParsingUtils.ParseLabelsFromLine(line); + if (!TestHelpers.StringListsAreEqual(parsedLabelsList, expectedLabelsList)) + { + string expectedLabelsForError = "Empty List"; + string parsedLabelsForError = "Empty List"; + if (expectedLabelsList.Count > 0) + { + expectedLabelsForError = string.Join(",", expectedLabelsList); + } + if (parsedLabelsList.Count > 0) + { + parsedLabelsForError = string.Join(",", parsedLabelsList); + } + Assert.Fail($"ParseLabelsFromLine for '{line}' should have returned {expectedLabelsForError} but instead returned {parsedLabelsForError}"); + } + } + + /// + /// Test ParsingUtils.ParseOwnersFromLine for linter. The linter doesn't expand teams because it needs + /// to verify the team entries. + /// + /// The CODEOWNERS line to parse + /// Expected list of owners to be parsed which includes team entries. + [Category("SourceOwners")] + [Category("Parsing")] + // source path/owner line with only users + [TestCase($"/sdk/FakePath1 @{TestHelpers.TestOwnerNamePartial}0 @{TestHelpers.TestOwnerNamePartial}4", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}4")] + // source path/owner line with users and a team should return the user and the team + [TestCase($"/sdk/FakePath2 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2\t@{TestHelpers.TestOwnerNamePartial}0", + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2", + $"{TestHelpers.TestOwnerNamePartial}0")] + // Case where a moniker has no owners + [TestCase($"# {MonikerConstants.AzureSdkOwners}:")] + // Again, using the SeparatorConstant.Owner instead of '@' would be ideal but NUnit won't + // allow the character constant to be within the string declaration. + [TestCase($"# {MonikerConstants.ServiceOwners}: @{TestHelpers.TestOwnerNamePartial}0\t@{TestHelpers.TestOwnerNamePartial}4", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}4")] + [TestCase($"# {MonikerConstants.ServiceOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}3\t@{TestHelpers.TestOwnerNamePartial}4", + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}3", + $"{TestHelpers.TestOwnerNamePartial}4")] + [TestCase($"#{MonikerConstants.MissingFolder}: @{TestHelpers.TestOwnerNamePartial}1\t@{TestHelpers.TestOwnerNamePartial}2", + $"{TestHelpers.TestOwnerNamePartial}1", + $"{TestHelpers.TestOwnerNamePartial}2")] + public void TestParseOwnersFromLineForLinter(string line, params string[] expectedOwners) + { + // Convert the array to List + var expectedOwnersList = expectedOwners.ToList(); + var parsedOwnersList = ParsingUtils.ParseOwnersFromLine(_ownerDataUtils, line, false /* linter doesn't expand teams */ ); + if (!TestHelpers.StringListsAreEqual(parsedOwnersList, expectedOwnersList)) + { + string expectedOwnersForError = "Empty List"; + string parsedOwnersForError = "Empty List"; + if (expectedOwnersList.Count > 0) + { + expectedOwnersForError = string.Join(",", expectedOwnersList); + } + if (parsedOwnersList.Count > 0) + { + parsedOwnersForError = string.Join(",", parsedOwnersList); + } + Assert.Fail($"ParseOwnersFromLine for '{line}' should have returned {expectedOwnersForError} but instead returned {parsedOwnersForError}"); + } + } + + /// + /// Test ParsingUtils.ParseOwnersFromLine for the parser, which does expand the teams and returns + /// only the distinct list of users. + /// + /// The CODEOWNERS line to parse + /// Expected list of owners to be parsed which includes owners from expanded teams. + [Category("SourceOwners")] + [Category("Parsing")] + // source path/owner line with only users + [TestCase($"/sdk/FakePath1 @{TestHelpers.TestOwnerNamePartial}0 @{TestHelpers.TestOwnerNamePartial}4", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}4")] + // source path/owner line with users and team with no intersection. + [TestCase($"/sdk/FakePath2 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2\t@{TestHelpers.TestOwnerNamePartial}4", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}1", + $"{TestHelpers.TestOwnerNamePartial}2", + $"{TestHelpers.TestOwnerNamePartial}4")] + // moniker line with users that is also in the team + [TestCase($"# {MonikerConstants.ServiceOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2\t@{TestHelpers.TestOwnerNamePartial}2", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}1", + $"{TestHelpers.TestOwnerNamePartial}2")] + // moniker line with only teams that have overlapping users + [TestCase($"# {MonikerConstants.AzureSdkOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}4", + $"{TestHelpers.TestOwnerNamePartial}0", + $"{TestHelpers.TestOwnerNamePartial}1", + $"{TestHelpers.TestOwnerNamePartial}2", + $"{TestHelpers.TestOwnerNamePartial}3", + $"{TestHelpers.TestOwnerNamePartial}4")] + // Case where a moniker has no owners + [TestCase($"# {MonikerConstants.ServiceOwners}:")] + public void TestParseOwnersFromLineForParser(string line, params string[] expectedOwners) + { + // Convert the array to List + var expectedOwnersList = expectedOwners.ToList(); + var parsedOwnersList = ParsingUtils.ParseOwnersFromLine(_ownerDataUtils, line, true /* parser will always expand teams */ ); + if (!TestHelpers.StringListsAreEqual(parsedOwnersList, expectedOwnersList)) + { + string expectedOwnersForError = "Empty List"; + string parsedOwnersForError = "Empty List"; + if (expectedOwnersList.Count > 0) + { + expectedOwnersForError = string.Join(",", expectedOwnersList); + } + if (parsedOwnersList.Count > 0) + { + parsedOwnersForError = string.Join(",", parsedOwnersList); + } + Assert.Fail($"ParseOwnersFromLine for '{line}' should have returned {expectedOwnersForError} but instead returned {parsedOwnersForError}"); + } + } + + /// + /// Test the Parsing.IsMonikerOrSourceLine. Anything that isn't a moniker or source line is effectively + /// ignored when parsing. + /// + /// The CODEOWNERS line to test. + /// Whether or not the line being tested should be detected as a moniker or source line. + // Source line with owners + [TestCase("/fakePath1/fakePath2 @fakeOwner1 @fakeOwner2", true)] + // Source line with no owners + [TestCase("/fakePath1/fakePath2", true)] + // Monikers with users or labels + [TestCase($"# {MonikerConstants.AzureSdkOwners}: @fakeOwner1 @fakeOwner2", true)] + [TestCase($"#{MonikerConstants.MissingFolder} @fakeOwner1 @fakeOwner2", true)] + [TestCase($"# {MonikerConstants.PRLabel}: %Fake Label", true)] + [TestCase($"# {MonikerConstants.ServiceLabel}: %Fake Label", true)] + [TestCase($"# {MonikerConstants.ServiceOwners}:", true)] + // Moniker only lines, without their owners or labels, still need to be positively identified + [TestCase($"# {MonikerConstants.AzureSdkOwners}:", true)] + [TestCase($"#{MonikerConstants.MissingFolder}", true)] + [TestCase($"# {MonikerConstants.PRLabel}:", true)] + [TestCase($"# {MonikerConstants.ServiceLabel}:", true)] + [TestCase($"# {MonikerConstants.ServiceOwners}:", true)] + // Some random lines that should not be detected as moniker or source lines + [TestCase(" \t", false)] + [TestCase("", false)] + [TestCase("# Just a comment line", false)] + // If the comment has a moniker in it but doesn't have the end colon, for the monikers that require + // them, then it shouldn't be detected as a moniker line. Note: This obviously will not work for + // the MissingFolder moniker which doesn't end with a colon but should for every other moniker. + [TestCase($"# {MonikerConstants.PRLabel} isn't moniker line (missing colon), but a comment with a moniker in it", false)] + public void TestIsMonikerOrSourceLine(string line, bool expectMonikerOrSourceLine) + { + bool isMonikerOrSourceLine = ParsingUtils.IsMonikerOrSourceLine(line); + Assert.That(expectMonikerOrSourceLine, Is.EqualTo(isMonikerOrSourceLine), $"IsMonikerOrSourceLine for {line} should have returned {expectMonikerOrSourceLine}"); + } + + /// + /// Tests to verify that IsSourcePathOwnerLine correctly identifies source path/owner lines. + /// A CODEOWNERS line is considered to be a source path/owner line if the line isn't a comment + /// line and isn't blank/whitespace. + /// + /// The line of source to verify + /// true if the line should verify as a source/path owner line, false otherwise. + [Category("CodeownersLinter")] + [Category("Verification")] + // Path with no owners + [TestCase("/someDir/someSubDir", true)] + // Path with owners + [TestCase("/someDir/someSubDir @owner1 @owner2", true)] + // Blank/Whitespace tests + [TestCase("", false)] + [TestCase(" ", false)] + [TestCase("\t", false)] + [TestCase("\r", false)] + [TestCase("\r\n", false)] + // Comment/Moniker lines + [TestCase("# it doesn't actually matter, it starts with a comment", false)] + [TestCase($"#{MonikerConstants.PRLabel}", false)] + [TestCase($"#{MonikerConstants.MissingFolder}", false)] + public void TestIsSourcePathOwnerLine(string line, bool expectSourcePathOwnerLine) + { + bool isSourcePathOwnerLine = ParsingUtils.IsSourcePathOwnerLine(line); + // If the line is expected to be a source path/owner line but the function says it isn't + if (expectSourcePathOwnerLine) + { + Assert.That(isSourcePathOwnerLine, Is.True, $"line '{line}' should have verified as a source path/owner line"); + } + else + { + Assert.That(isSourcePathOwnerLine, Is.False, $"line '{line}' should not have verified as a source path/owner line"); + } + } + + /// + /// Test the FindBlockEnd function. There are a couple of things of note here: + /// 1. Each test scenario should have its own CODEOWNERS file in Tests.FakeCodeowners/FindBlockEndTests. + /// This will allow tests to run in parallel and prevent any updates or changes to one test from + /// affectiing every test. + /// 2. If there are specific notes required for the scenario, they should be comments in the file. + /// 3. A block must start with a non-blank line and will end in one of the following ways: + /// a. A blank line + /// b. A source path/owner line + /// c. The end of the file + /// Note: Editing the Codeowners test files in VS shows line numbers as 1 based but the lines + /// are actually 0 based for processing. + /// + /// The test codeowners file + /// The line number of the block start + /// The expected line end number + [TestCase("CodeownersTestFiles/FindBlockEnd/SingleSourcePathLine", 0, 0)] + [TestCase("CodeownersTestFiles/FindBlockEnd/PRLabelAndSourcePath", 1, 2)] + [TestCase("CodeownersTestFiles/FindBlockEnd/ServiceLabelAndSourcePath", 0, 2)] + [TestCase("CodeownersTestFiles/FindBlockEnd/ServiceLabelAndMissingFolder", 1, 2)] + [TestCase("CodeownersTestFiles/FindBlockEnd/AllMonikersEndsWithSourcePath", 1, 5)] + public void TestFindBlockEnd(string testCodeownerFile, int startBlockLineNumber, int expectedEndBlockLineNumber) + { + List codeownersFile = FileHelpers.LoadFileAsStringList(testCodeownerFile); + int actualEndBlockLineNumber = ParsingUtils.FindBlockEnd(startBlockLineNumber, codeownersFile); + Assert.That(actualEndBlockLineNumber, Is.EqualTo(expectedEndBlockLineNumber), $"The expected end line number for {testCodeownerFile} was {expectedEndBlockLineNumber} but the actual end line number was {actualEndBlockLineNumber}"); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/RepoLabelDataUtilsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/RepoLabelDataUtilsTests.cs new file mode 100644 index 00000000000..c27e1164d7f --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Utils/RepoLabelDataUtilsTests.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Utils +{ + /// + /// OwnerDataUtilsTests requires a RepoLabelDataUtils with populated RepoLabelCache + /// + [TestFixture] + [Parallelizable(ParallelScope.Children)] + public class RepoLabelDataUtilsTests + { + private RepoLabelDataUtils _repoLabelDataUtils; + + [OneTimeSetUp] + public void InitRepoLabelData() + { + _repoLabelDataUtils = TestHelpers.SetupRepoLabelData(); + // None of the tests will work if the repo/label data doesn't exist. + // While the previous function call created the test repo/label data, + // this call just ensures no changes were made to the util that'll + // mess things up. + if (!_repoLabelDataUtils.RepoLabelDataExists()) + { + throw new ArgumentException($"Test repo/label data should have been created for {TestHelpers.TestRepositoryName} but was not."); + } + } + + /// + /// Test whether or not the label exists in the repo/label data. The lookup + /// for both the label and the repository need to be setup case insensitive + /// since GitHub is case insensitive but case preserving. + /// + /// The label to test. + /// True if the label should exist, false otherwise. + [Category("Utils")] + [Category("Labels")] + [TestCase($"{TestHelpers.TestLabelNamePartial}0", true)] + [TestCase($"{TestHelpers.TestLabelNamePartial}4", true)] + [TestCase($"{TestHelpers.TestOwnerNamePartial}5", false)] + public void TestLabelInRepo(string label, bool expectedResult) + { + // Test owner lookup with preserved case owner + bool isLabelInRepo = _repoLabelDataUtils.LabelInRepo(label); + Assert.That(isLabelInRepo, Is.EqualTo(expectedResult), $"LabelInRepo for {label} should have returned {expectedResult} and did not"); + + // Test owner lookup with uppercase owner + string labelUpper = label.ToUpper(); + isLabelInRepo = _repoLabelDataUtils.LabelInRepo(labelUpper); + Assert.That(isLabelInRepo, Is.EqualTo(expectedResult), $"LabelInRepo for {labelUpper} should have returned {expectedResult} and did not"); + + // Test owner lookup with lowercase owner + string labelLower = label.ToLower(); + isLabelInRepo = _repoLabelDataUtils.LabelInRepo(labelLower); + Assert.That(isLabelInRepo, Is.EqualTo(expectedResult), $"LabelInRepo for {labelLower} should have returned {expectedResult} and did not"); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/CodeownersLinterTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/CodeownersLinterTests.cs new file mode 100644 index 00000000000..854ad675621 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/CodeownersLinterTests.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Tests.Mocks; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Verification +{ + /// + /// Tests for CodeownersLinter. These tests require the following: + /// 1. OwnerDataUtils with populated team/user and user/org visibility data + /// 2. RepoLabelDataUtils with populated RepoLabelCache + /// 3. A mock DirectoryUtils that doesn't actually do directory verification. + /// + [TestFixture] + [Parallelizable(ParallelScope.Children)] + public class CodeownersLinterTests + { + private OwnerDataUtils _ownerDataUtils; + private RepoLabelDataUtils _repoLabelDataUtils; + private DirectoryUtilsMock _directoryUtilsMock; + + [OneTimeSetUp] + // Initialize a DirectoryUtilsMock, OwnerDataUtils and RepoLabelDataUtils + public void InitTestData() + { + _directoryUtilsMock = new DirectoryUtilsMock(); + _ownerDataUtils = TestHelpers.SetupOwnerData(); + _repoLabelDataUtils = TestHelpers.SetupRepoLabelData(); + // None of the tests will work if the repo/label data doesn't exist. + // While the previous function call created the test repo/label data, + // this call just ensures no changes were made to the util that'll + // mess things up. + if (!_repoLabelDataUtils.RepoLabelDataExists()) + { + throw new ArgumentException($"Test repo/label data should have been created for {TestHelpers.TestRepositoryName} but was not."); + } + } + + /// + /// Test the VerifySingleLine function. Note that all the parse and verify functions are tested + /// in the Labels and SourceOwners tests. Any expected actualErrors are just to ensure that the SingleLineError + /// created correctly contains all of the strings. + /// + /// The CODEOWNERS line to verify + /// True if the line is a source path/owner line. + /// True if the line is a moniker line and owners are expected. This is the case where certain monikers require owners if the block doesn't end in source path/owner line. + /// The moniker, if the line is a moniker line, null otherwise. + /// Expected error strings on the SingleLineError, if any. + [Category("CodeownersLinter")] + [Category("Verification")] + //**source path/owner line scenarios** + // valid team and owner + [TestCase($"/sdk/subDir1/subDir1 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}1 @{TestHelpers.TestOwnerNamePartial}2", + true, + false, + null)] + // malformed team entry (missing the @Azure/) + [TestCase($"/sdk/subDir1/subDir1 @{TestHelpers.TestTeamNamePartial}1", + true, + false, + null, + $"{TestHelpers.TestTeamNamePartial}1{ErrorMessageConstants.MalformedTeamEntryPartial}")] + // invalid team, invalid user, non-public user, public user and a valid team + [TestCase($"/sdk/subDir1/subDir1 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}54\t@{TestHelpers.TestOwnerNamePartial}6 @{TestHelpers.TestOwnerNamePartial}2\t@{TestHelpers.TestOwnerNamePartial}3 @Azure/{TestHelpers.TestTeamNamePartial}3", + true, + false, + null, + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}54{ErrorMessageConstants.InvalidTeamPartial}", + $"{TestHelpers.TestOwnerNamePartial}6{ErrorMessageConstants.InvalidUserPartial}", + $"{TestHelpers.TestOwnerNamePartial}3{ErrorMessageConstants.NotAPublicMemberOfAzurePartial}")] + //**Moniker owner scenarios** + // valid team and valid user + [TestCase($"# {MonikerConstants.ServiceOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}4 @{TestHelpers.TestOwnerNamePartial}2", + false, + true, + MonikerConstants.ServiceOwners)] + [TestCase($"#{MonikerConstants.MissingFolder}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}2 @{TestHelpers.TestOwnerNamePartial}0", + false, + true, + MonikerConstants.ServiceOwners)] + // invalid team, invalid user, non-public user, public user and a valid team + [TestCase($"# {MonikerConstants.AzureSdkOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}54\t@{TestHelpers.TestOwnerNamePartial}6 @{TestHelpers.TestOwnerNamePartial}2\t@{TestHelpers.TestOwnerNamePartial}3 @Azure/{TestHelpers.TestTeamNamePartial}3", + false, + true, + MonikerConstants.AzureSdkOwners, + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}54{ErrorMessageConstants.InvalidTeamPartial}", + $"{TestHelpers.TestOwnerNamePartial}6{ErrorMessageConstants.InvalidUserPartial}", + $"{TestHelpers.TestOwnerNamePartial}3{ErrorMessageConstants.NotAPublicMemberOfAzurePartial}")] + //**Moniker label scenarios** + // valid labels with and without % signs. The old style used % sign as the delimiter but only really required + // two labels for ServiceLabel entries because of the Service Attention label. The new style just assumes everything + // after the : (trimmed, of course) is the label + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}0", + false, + false, + MonikerConstants.PRLabel)] + [TestCase($"# {MonikerConstants.PRLabel}: {TestHelpers.TestLabelNamePartial}2", + false, + false, + MonikerConstants.PRLabel)] + // ServiceLabel can only have two labels if one of them is ServiceAttention + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}1 %{LabelConstants.ServiceAttention}", + false, + false, + MonikerConstants.ServiceLabel)] + [TestCase($"# {MonikerConstants.ServiceLabel}: {TestHelpers.TestLabelNamePartial}4", + false, + false, + MonikerConstants.ServiceLabel)] + // ServiceAttention is not a valid PRLabel + [TestCase($"# {MonikerConstants.PRLabel}: {LabelConstants.ServiceAttention}", + false, + false, + MonikerConstants.PRLabel, + ErrorMessageConstants.ServiceAttentionIsNotAValidPRLabel)] + // Two PRLabels and one of them is invalid + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}987 %{TestHelpers.TestLabelNamePartial}4", + false, + false, + MonikerConstants.PRLabel, + $"'{TestHelpers.TestLabelNamePartial}987'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // Too many labels for ServiceLabel. Three labels and one of them is ServiceAttention and one is invalid + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}2\t%{TestHelpers.TestLabelNamePartial}345 %{LabelConstants.ServiceAttention}", + false, + false, + MonikerConstants.ServiceLabel, + $"'{TestHelpers.TestLabelNamePartial}345'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // ServiceLabel with only ServiceAttention is an error + [TestCase($"# {MonikerConstants.ServiceLabel}: %{LabelConstants.ServiceAttention}", + false, + false, + MonikerConstants.ServiceLabel, + ErrorMessageConstants.ServiceLabelMustContainAServiceLabel)] + public void TestVerifySingleLine(string line, + bool isSourcePathOwnerLine, + bool expectOwnersIfMoniker, + string moniker, + params string[] expectedErrorMessages) + { + // Convert the array to List + var expectedErrorMessagesList = expectedErrorMessages.ToList(); + // The line number for reporting doesn't matter for the testcases except to + // check, if there are any actualErrors, that the line number is set correctly. + int lineNumberForReporting = 42; + List actualErrors = new List(); + CodeownersLinter.VerifySingleLine(_directoryUtilsMock, + _ownerDataUtils, + _repoLabelDataUtils, + actualErrors, + lineNumberForReporting, + line, + isSourcePathOwnerLine, + expectOwnersIfMoniker, + moniker); + // Check and see if there were any actualErrors returned and whether or not an error was expected. + if (expectedErrorMessagesList.Count == 0) + { + // The number of actualErrors from VerifySingleLine will always be one SingleLineError + // with one or more error strings + if (actualErrors.Count > 0) + { + string actualErrorsString = string.Join(Environment.NewLine, actualErrors[0].Errors); + Assert.Fail($"VerifySingleLine for {line} had no expected errors but returned\n{actualErrorsString}"); + } + } + // Test expects errors + else + { + if (actualErrors.Count == 0) + { + string expectedErrorsString = string.Join(Environment.NewLine, expectedErrorMessagesList); + Assert.Fail($"VerifySingleLine for {line} did not produce the any errors but should have had the following errors\n{expectedErrorsString}"); + } + else + { + if (!TestHelpers.StringListsAreEqual(expectedErrorMessagesList, actualErrors[0].Errors)) + { + string expectedErrorsString = string.Join(Environment.NewLine, expectedErrorMessagesList); + string actualErrorsString = string.Join(Environment.NewLine, actualErrors[0].Errors); + Assert.Fail($"VerifySingleLine for {line} should have had the following errors\n{expectedErrorsString}\nbut instead had\n{actualErrorsString}"); + } + } + } + } + + /// + /// Test Block Verification. The purpose here isn't to test owners or labels but rather to + /// test the contents of the block to ensure its completeness. For example, the PRLabel + /// moniker requires that the block ends in a source path/owner line. Or a ServiceLabel + /// must be paired with ServiceOwners, NotInRepo or be part of a block that ends in a + /// source path/owner line. + /// + /// The test codeowners file + /// The start line number of the block + /// The end line number of the block + /// Expected error messages, if any + [Category("CodeownersLinter")] + [Category("Verification")] + // Success cases, these files shouldn't produce any errors + [TestCase("CodeownersTestFiles/VerifyBlock/SingleSourcePathLine", 1, 1)] + [TestCase("CodeownersTestFiles/VerifyBlock/PRLabelAndSourcePath", 1, 2)] + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelAndSourcePath", 1, 2)] + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelAndMissingPath", 1, 2)] + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelAndServiceOwners", 1, 2)] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersEndsInSourcePath", 1, 4)] + // AzureSdkOwners needs to be part of a block that contains ServiceLabel + [TestCase("CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel", 3, 3, + ErrorMessageConstants.AzureSdkOwnersMustBeWithServiceLabel)] + [TestCase("CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel", 7, 7, + ErrorMessageConstants.AzureSdkOwnersMustBeWithServiceLabel)] + [TestCase("CodeownersTestFiles/VerifyBlock/AzureSdkOwnersAndServiceLabel", 9, 9, + ErrorMessageConstants.AzureSdkOwnersMustBeWithServiceLabel)] + // Monikers that need to be in a block that ends in a source/path + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 1, 1, + $"{MonikerConstants.PRLabel}{ErrorMessageConstants.NeedsToEndWithSourceOwnerPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 5, 5, + $"{MonikerConstants.PRLabel}{ErrorMessageConstants.NeedsToEndWithSourceOwnerPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 7, 7, + $"{MonikerConstants.PRLabel}{ErrorMessageConstants.NeedsToEndWithSourceOwnerPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 11, 11, + ErrorMessageConstants.ServiceLabelNeedsOwners)] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 15, 15, + ErrorMessageConstants.ServiceLabelNeedsOwners)] + [TestCase("CodeownersTestFiles/VerifyBlock/MonikersMissingSourcePath", 17, 17, + ErrorMessageConstants.ServiceLabelNeedsOwners)] + // Duplicate Moniker Errors + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 3, 5, + $"{MonikerConstants.PRLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 8, 11, + $"{MonikerConstants.AzureSdkOwners}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 14, 16, + $"{MonikerConstants.ServiceLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 19, 21, + $"{MonikerConstants.ServiceLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 24, 26, + $"{MonikerConstants.ServiceLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 29, 31, + $"{MonikerConstants.MissingFolder}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + [TestCase("CodeownersTestFiles/VerifyBlock/DuplicateMonikers", 34, 36, + $"{MonikerConstants.ServiceOwners}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}")] + // ServiceLabel ends in source path/owner line with ServiceOwners or MissingPath, //, in the same block + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers", 2, 4, + ErrorMessageConstants.ServiceLabelHasTooManyOwners)] + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers", 8, 10, + ErrorMessageConstants.ServiceLabelHasTooManyOwners)] + // ServiceLabel is part of a block that has both ServiceOwners and MissingPath, //. + [TestCase("CodeownersTestFiles/VerifyBlock/ServiceLabelTooManyOwnersAndMonikers", 13, 15, + ErrorMessageConstants.ServiceLabelHasTooManyOwnerMonikers)] + public void TestVerifyBlock(string testCodeownerFile, + int startBlockLineNumber, + int endBlockLineNumber, + params string[] expectedErrorMessages) + { + // Convert the array to List + var expectedErrorMessagesList = expectedErrorMessages.ToList(); + // Load the codeowners file + List codeownersFile = FileHelpers.LoadFileAsStringList(testCodeownerFile); + List returnedErrors = new List(); + CodeownersLinter.VerifyBlock(_directoryUtilsMock, + _ownerDataUtils, + _repoLabelDataUtils, + returnedErrors, + startBlockLineNumber, + endBlockLineNumber, + codeownersFile); + + // Ensure that the actual error list only contains BlockFormatting errors. For example, + // an AzureSdkOwners needs to be in a block that ends in a source/path owner line whether + // or not it has owners defined but, without owners defined and not being part of a block + // which ends in a source path/owner line, it'll cause a SingleLineError (for no owners) as + // well as the BlockFormattingError. The SingleLineErrors are checked elsewhere and only + // BlockFormattingErrors matter for these testcases. + var blockFormattingErrors = returnedErrors.OfType().ToList(); + + // Check and see if there were any actualErrors returned and whether or not an error was expected. + if (expectedErrorMessagesList.Count == 0) + { + // The number of actualErrors from VerifySingleLine will always be one SingleLineError + // with one or more error strings + if (blockFormattingErrors.Count > 0) + { + string actualErrorsString = string.Join(Environment.NewLine, blockFormattingErrors[0].Errors); + Assert.Fail($"VerifyBlock for {testCodeownerFile}, start/end lines {startBlockLineNumber}/{endBlockLineNumber}, had no expected errors but returned\n{actualErrorsString}"); + } + } + // Test expects errors + else + { + if (blockFormattingErrors.Count == 0) + { + string expectedErrorsString = string.Join(Environment.NewLine, expectedErrorMessagesList); + Assert.Fail($"VerifyBlock for {testCodeownerFile}, start/end lines {startBlockLineNumber}/{endBlockLineNumber}, did not produce the any errors but should have had the following errors\n{expectedErrorsString}"); + } + else + { + if (!TestHelpers.StringListsAreEqual(expectedErrorMessagesList, blockFormattingErrors[0].Errors)) + { + string expectedErrorsString = string.Join(Environment.NewLine, expectedErrorMessagesList); + string actualErrorsString = string.Join(Environment.NewLine, blockFormattingErrors[0].Errors); + Assert.Fail($"VerifyBlock for {testCodeownerFile}, start/end lines {startBlockLineNumber}/{endBlockLineNumber}, should have had the following errors\n{expectedErrorsString}\nbut instead had\n{actualErrorsString}"); + } + } + } + } + + /// + /// End to end tests. All the individual pieces parts have all been tested, these + /// are just the end to end pieces. The test(s) with errors are going to use an + /// existing baseline file for verification if errors are expected. Additionally, + /// they'll also generate a baseline file re-verify with that. + /// + /// The file that contains the CODEOWNERS data for a given scenario. + /// The file that contains the expected baseline errors for a given scenario. + [TestCase("CodeownersTestFiles/EndToEnd/NoErrors", null, 0)] + [TestCase("CodeownersTestFiles/EndToEnd/WithBlockErrors", "CodeownersTestFiles/EndToEnd/WithBlockErrors_baseline.txt", 4)] + public void TestLintCodeownersFile(string testCodeownerFile, + string testBaselineFile, + int expectedNumberOfBlockErrors) + { + List actualErrors = CodeownersLinter.LintCodeownersFile(_directoryUtilsMock, + _ownerDataUtils, + _repoLabelDataUtils, + testCodeownerFile); + // If errors weren't expected... + if (testBaselineFile == null) + { + // ...but were produced + if (actualErrors.Count > 0) + { + string errorString = TestHelpers.FormatErrorMessageFromErrorList(actualErrors); + Assert.Fail($"LintCodeownersFile for {testCodeownerFile} should not have produced any errors but had {actualErrors.Count} errors.\nErrors:\n{errorString}"); + } + } + // If errors are expected... + else + { + // ...but none were produced + if (actualErrors.Count == 0) + { + Assert.Fail($"LintCodeownersFile for {testCodeownerFile} expected errors, testBaselineFile={testBaselineFile}, but did not produce any."); + } + else + { + // Last but not least, make sure that any SingleLineErrors produced match what is expected. To do this, the baseline file + // and filtering will be used. + BaselineUtils baselineUtils = new BaselineUtils(testBaselineFile); + var filteredErrors = baselineUtils.FilterErrorsUsingBaseline(actualErrors); + // The filter file contains all of the expected SingleLineErrors errors. After filtering, the list of filtered + // errors should only contain the expected number of BlockFormattingErrors. + if (filteredErrors.Count != expectedNumberOfBlockErrors) + { + string failureString = $"LintCodeownersFile for {testCodeownerFile} expected {expectedNumberOfBlockErrors} of block formatting errors but returned {filteredErrors.Count} errors."; + // Are all of the errors block errors? + var singleLineErrors = filteredErrors.OfType().ToList(); + if (singleLineErrors.Count > 0) + { + failureString += $"\nOf the failures, there were {singleLineErrors.Count} errors that should have been filtered."; + } + + string errorString = TestHelpers.FormatErrorMessageFromErrorList(filteredErrors); + Assert.Fail($"{failureString}.\nFiltered Errors:\n{errorString}."); + } + } + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/LabelsTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/LabelsTests.cs new file mode 100644 index 00000000000..3856884896a --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/LabelsTests.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Verification +{ + /// + /// Tests for label parsing and verification. LabelsTests requires a RepoLabelDataUtils with populated RepoLabelCache + /// + [TestFixture] + [Parallelizable(ParallelScope.Children)] + + public class LabelsTests + { + private RepoLabelDataUtils _repoLabelDataUtils; + + [OneTimeSetUp] + public void InitRepoLabelData() + { + _repoLabelDataUtils = TestHelpers.SetupRepoLabelData(); + // None of the tests will work if the repo/label data doesn't exist. + // While the previous function call created the test repo/label data, + // this call just ensures no changes were made to the util that'll + // mess things up. + if (!_repoLabelDataUtils.RepoLabelDataExists()) + { + throw new ArgumentException($"Test repo/label data should have been created for {TestHelpers.TestRepositoryName} but was not."); + } + } + + /// + /// Given a CODEOWNERS line with monikers that expect labels, parse the labels and + /// verify that the labels exist in the repository. + /// + /// The CODEOWNERS line to parse. Note that service labels with and without % seperator + /// Expected error messages, if the line contains errors + [Category("Labels")] + [Category("Verification")] + // PRLabel and Service Label with no errors with {SeparatorConstants.Label} before the label(s) + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}0", + MonikerConstants.PRLabel)] + [TestCase($"# {MonikerConstants.ServiceLabel}:\t%{TestHelpers.TestLabelNamePartial}1", + MonikerConstants.ServiceLabel)] + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}2\t%{LabelConstants.ServiceAttention}", + MonikerConstants.ServiceLabel)] + // PRLabel and Service Label with no errors without % before the label. This is from the new syntax + // where everything after the :, trimmed, is treated as the label + [TestCase($"# {MonikerConstants.PRLabel}:\t{TestHelpers.TestLabelNamePartial}4", + MonikerConstants.PRLabel)] + [TestCase($"# {MonikerConstants.ServiceLabel}: {TestHelpers.TestLabelNamePartial}0", + MonikerConstants.ServiceLabel)] + // PRLabel and Service Label with no label + [TestCase($"# {MonikerConstants.PRLabel}:", + MonikerConstants.PRLabel, + $"{ErrorMessageConstants.MissingLabelForMoniker}")] + [TestCase($"# {MonikerConstants.ServiceLabel}:", + MonikerConstants.ServiceLabel, + $"{ErrorMessageConstants.MissingLabelForMoniker}")] + // PRLabel with an invalid label for the repository + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}567", + MonikerConstants.PRLabel, + $"'{TestHelpers.TestLabelNamePartial}567'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // ServiceLabel with an invalid label for the repository + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}567 %{LabelConstants.ServiceAttention}", + MonikerConstants.ServiceLabel, + $"'{TestHelpers.TestLabelNamePartial}567'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // Verify that ServiceAttention on a PRLabel is reported as an error + [TestCase($"# {MonikerConstants.PRLabel}: {LabelConstants.ServiceAttention}", + MonikerConstants.PRLabel, + ErrorMessageConstants.ServiceAttentionIsNotAValidPRLabel)] + // Verify that an invalid label and ServiceAttention on a PRLabel moniker reports "ServiceAttention is not a valid PRLabel" and that + // the label is not valid for the repository. + [TestCase($"# {MonikerConstants.PRLabel}: %{TestHelpers.TestLabelNamePartial}55 %{LabelConstants.ServiceAttention}", + MonikerConstants.PRLabel, + ErrorMessageConstants.ServiceAttentionIsNotAValidPRLabel, + $"'{TestHelpers.TestLabelNamePartial}55'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // Service Label moniker with more than 2 labels is reported as an error. Labels are still verified and any of those errors are also reported. + [TestCase($"# {MonikerConstants.ServiceLabel}: %{TestHelpers.TestLabelNamePartial}0 %{TestHelpers.TestLabelNamePartial}55 %{TestHelpers.TestLabelNamePartial}3", + MonikerConstants.ServiceLabel, + $"'{TestHelpers.TestLabelNamePartial}55'{ErrorMessageConstants.InvalidRepositoryLabelPartial}")] + // Service Label moniker with only ServiceAttention is an error + [TestCase($"# {MonikerConstants.ServiceLabel}: {LabelConstants.ServiceAttention}", + MonikerConstants.ServiceLabel, + ErrorMessageConstants.ServiceLabelMustContainAServiceLabel)] + public void TestVerifyLabels(string line, string moniker, params string[] expectedErrorMessages) + { + var expectedErrorList = expectedErrorMessages.ToList(); + List actualErrorList = new List(); + Labels.VerifyLabels(_repoLabelDataUtils, line, moniker, actualErrorList); + if (!TestHelpers.StringListsAreEqual(actualErrorList, expectedErrorList)) + { + string expectedErrors = "Empty List"; + string actualErrors = "Empty List"; + if (expectedErrorList.Count > 0) + { + expectedErrors = string.Join(Environment.NewLine, expectedErrorList); + } + if (actualErrorList.Count > 0) + { + actualErrors = string.Join(Environment.NewLine, actualErrorList); + } + Assert.Fail($"VerifyLabels for '{line}' should have returned:\n{expectedErrors}\nbut instead returned\n{actualErrors}"); + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/OwnersTests.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/OwnersTests.cs new file mode 100644 index 00000000000..b32ca4e39e4 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter.Tests/Verification/OwnersTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.CodeownersUtils.Tests.Verification +{ + /// + /// Tests for owners parsing and varification. OwnersTests requires a OwnerDataUtils with populated team/user and user/org visibility data + /// + [TestFixture] + [Parallelizable(ParallelScope.Children)] + + public class OwnersTests + { + private OwnerDataUtils _ownerDataUtils; + + [OneTimeSetUp] + public void InitRepoLabelData() + { + _ownerDataUtils = TestHelpers.SetupOwnerData(); + } + + /// + /// Test Owners.VerifyOwners. It's worth noting that VerifyOwners validates individual owners and teams on a given line and + /// does not expand teams. + /// + /// The CODEOWNERS line to parse. + /// Whether or not owners are expected. Some monikers may or may not have owners if their block ends in a source path/owner line and some source/path owner lines might not have owners. + /// If owners are expected, the expected list of owners to be parsed. + [Category("SourceOwners")] + [Category("Verification")] + // Source path/owner line with no errors + [TestCase($"/sdk/SomePath @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}0\t@{TestHelpers.TestOwnerNamePartial}2",true)] + // Moniker Owner lines with no errors + [TestCase($"# {MonikerConstants.AzureSdkOwners}: @{TestHelpers.TestOwnerNamePartial}0 @{TestHelpers.TestOwnerNamePartial}4", true)] + [TestCase($"# {MonikerConstants.ServiceOwners}: @{TestHelpers.TestOwnerNamePartial}4 @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}1\t\t@{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}3", true)] + // AzureSdkOwners, with no owner defined is legal for a block that ends in a source path/owner line. + [TestCase($"# {MonikerConstants.AzureSdkOwners}:", false)] + // Source path/owner line with no owners should complain + [TestCase($"/sdk/SomePath", true, ErrorMessageConstants.NoOwnersDefined)] + // AzureSdkOwners, with no owner defined is not legal if the block doesn't end in a source path/owner line. + [TestCase($"# {MonikerConstants.AzureSdkOwners}:", true, ErrorMessageConstants.NoOwnersDefined)] + // At this point whether or not the line is a moniker or source path/owner line is irrelevant. + // Test each error individually. + // Invalid team + [TestCase($"# {MonikerConstants.ServiceOwners}: @{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}12", true, + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}12{ErrorMessageConstants.InvalidTeamPartial}")] + // Invalid User + [TestCase($"# {MonikerConstants.ServiceOwners}: @{TestHelpers.TestOwnerNamePartial}456", true, + $"{TestHelpers.TestOwnerNamePartial}456{ErrorMessageConstants.InvalidUserPartial}")] + // Non-public member + [TestCase($"# {MonikerConstants.ServiceOwners}: @{TestHelpers.TestOwnerNamePartial}1", true, + $"{TestHelpers.TestOwnerNamePartial}1{ErrorMessageConstants.NotAPublicMemberOfAzurePartial}")] + // Malformed team entry (missing @Azure/) but team otherwise exists in the azure-sdk-write dictionary + [TestCase($"/sdk/SomePath @{TestHelpers.TestTeamNamePartial}0", true, + $"{TestHelpers.TestTeamNamePartial}0{ErrorMessageConstants.MalformedTeamEntryPartial}")] + // All the owners errors on a single line (except no owners errors) + [TestCase($"/sdk/SomePath @{TestHelpers.TestTeamNamePartial}0\t@{TestHelpers.TestOwnerNamePartial}1 @{TestHelpers.TestOwnerNamePartial}456\t\t\t@{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}12", + true, + $"{TestHelpers.TestTeamNamePartial}0{ErrorMessageConstants.MalformedTeamEntryPartial}", + $"{TestHelpers.TestOwnerNamePartial}1{ErrorMessageConstants.NotAPublicMemberOfAzurePartial}", + $"{TestHelpers.TestOwnerNamePartial}456{ErrorMessageConstants.InvalidUserPartial}", + $"{OrgConstants.Azure}/{TestHelpers.TestTeamNamePartial}12{ErrorMessageConstants.InvalidTeamPartial}")] + public void TestVerifyOwners(string line, bool expectOwners, params string[] expectedErrorMessages) + { + // Convert the array to List + var expectedErrorList = expectedErrorMessages.ToList(); + List actualErrorList = new List(); + Owners.VerifyOwners(_ownerDataUtils, line, expectOwners, actualErrorList); + if (!TestHelpers.StringListsAreEqual(actualErrorList, expectedErrorList)) + { + string expectedErrors = "Empty List"; + string actualErrors = "Empty List"; + if (expectedErrorList.Count > 0) + { + expectedErrors = string.Join(Environment.NewLine, expectedErrorList); + } + if (actualErrorList.Count > 0) + { + actualErrors = string.Join(Environment.NewLine, actualErrorList); + } + Assert.Fail($"VerifyOwners for '{line}' should have returned:\n{expectedErrors}\nbut instead returned\n{actualErrors}"); + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Azure.Sdk.Tools.CodeownersLinter.csproj b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Azure.Sdk.Tools.CodeownersLinter.csproj new file mode 100644 index 00000000000..d6e967e6a2c --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Azure.Sdk.Tools.CodeownersLinter.csproj @@ -0,0 +1,18 @@ + + + + Exe + net6.0 + disable + codeowners-linter + + + + + + + + + + + diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Program.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Program.cs new file mode 100644 index 00000000000..ef832be081e --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersLinter/Program.cs @@ -0,0 +1,231 @@ +using System.CommandLine; +using System.Diagnostics; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Caches; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Verification; +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; + +namespace Azure.Sdk.Tools.CodeownersLinter +{ + internal class Program + { + static void Main(string[] args) + { + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); + + // The storage URIs are in the azure-sdk-write-teams-container-blobs pipeline variable group. + // The URIs do not contain the SAS. + var teamUserBlobStorageUriOption = new Option + (name: "--teamUserBlobStorageURI", + description: "The team/user blob storage URI without the SAS."); + teamUserBlobStorageUriOption.AddAlias("-tUri"); + teamUserBlobStorageUriOption.IsRequired = true; + + var userOrgVisibilityBlobStorageUriOption = new Option + (name: "--userOrgVisibilityBlobStorageURI", + description: "The user/org blob storage URI without the SAS."); + userOrgVisibilityBlobStorageUriOption.AddAlias("-uUri"); + userOrgVisibilityBlobStorageUriOption.IsRequired = true; + + var repoLabelBlobStorageUriOption = new Option + (name: "--repoLabelBlobStorageURI", + description: "The repo/label blob storage URI without the SAS."); + repoLabelBlobStorageUriOption.AddAlias("-rUri"); + repoLabelBlobStorageUriOption.IsRequired = true; + + // In a pipeline the repoRoot option should be as follows on the command line + // --repoRoot $(Build.SourcesDirectory) + var repoRootOption = new Option + (name: "--repoRoot", + description: "The root of the repository to be scanned."); + repoRootOption.IsRequired = true; + + // In a pipeline, the repo name option should be as follows on the command line + // --repoName $(Build.Repository.Name) + var repoNameOption = new Option + (name: "--repoName", + description: "The name of the repository."); + repoNameOption.IsRequired = true; + + // Whether or not to use the baseline. If this option is selected it'll + // load the baseline file that sits beside the CODEOWNERS file and only + // report errors that don't exist in the baseline. + // Note, this is flag and flags default to false if they're not on the command line. + var filterBaselineErrorsOption = new Option + (name: "--filterBaselineErrors", + description: "Only output errors that don't exist in the baseline."); + filterBaselineErrorsOption.AddAlias("-fbl"); + + var generateBaselineOption = new Option + (name: "--generateBaseline", + description: "Generate the baseline error file."); + generateBaselineOption.AddAlias("-gbl"); + + var rootCommand = new RootCommand + { + teamUserBlobStorageUriOption, + userOrgVisibilityBlobStorageUriOption, + repoLabelBlobStorageUriOption, + repoRootOption, + repoNameOption, + filterBaselineErrorsOption, + generateBaselineOption + }; + + int returnCode = 1; + // This might look a bit odd. System.CommandLine cannot have a non-async handler that isn't a void + // which means that the call only returns non-zero if the handler call fails. Instead of setting the + // handler to the function with the option arguments, the handler needs to take the context, grab + // all of the option values, call the function and set the local variable to the return value. + // Q) Why is this necessary? + // A) The linting of the CODEOWNERS file needs to be able to provide a return value for success and + // failure. The only way the handler returns a failure, in the non-async, case would be something + // like an unhandled exception. + rootCommand.SetHandler( + (context) => + { + string teamUserBlobStorageUri = context.ParseResult.GetValueForOption(teamUserBlobStorageUriOption); + string userOrgVisibilityBlobStorageUri = context.ParseResult.GetValueForOption(userOrgVisibilityBlobStorageUriOption); + string repoLabelBlobStorageUri = context.ParseResult.GetValueForOption(repoLabelBlobStorageUriOption); + string repoRoot = context.ParseResult.GetValueForOption(repoRootOption); + string repoName = context.ParseResult.GetValueForOption(repoNameOption); + bool filterBaselineErrors = context.ParseResult.GetValueForOption(filterBaselineErrorsOption); + bool generateBaseline = context.ParseResult.GetValueForOption(generateBaselineOption); + returnCode = LintCodeownersFile(teamUserBlobStorageUri, + userOrgVisibilityBlobStorageUri, + repoLabelBlobStorageUri, + repoRoot, + repoName, + filterBaselineErrors, + generateBaseline); + }); + + rootCommand.Invoke(args); + + stopWatch.Stop(); + TimeSpan ts = stopWatch.Elapsed; + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine($"Total run time: {elapsedTime}"); + + Console.WriteLine($"Exiting with return code {returnCode}"); + Environment.Exit(returnCode); + } + + /// + /// Verify the arguments and call to process the CODEOWNERS file. If errors are being filtered with a + /// baseline, or used to regenerate the baseline, that's done in here. Note that filtering errors and + /// regenerating the baseline cannot both be done in the same run. + /// + /// URI of the team/user data in blob storate + /// URI of the org visibility in blob storage + /// URI of the repository label data + /// The root of the repository + /// The repository name, including org. Eg. Azure/azure-sdk + /// Boolean, if true then errors should be filtered using the repository's baseline. + /// Boolean, if true then regenerate the baseline file from the error encountered during parsing. + /// integer, used to set the return code + /// Thrown if any arguments, or argument combinations, are invalid. + static int LintCodeownersFile(string teamUserBlobStorageUri, + string userOrgVisibilityBlobStorageUri, + string repoLabelBlobStorageUri, + string repoRoot, + string repoName, + bool filterBaselineErrors, + bool generateBaseline) + { + // Don't allow someone to create and use a baseline in the same run + if (filterBaselineErrors && generateBaseline) + { + throw new ArgumentException("The --filterBaselineErrors (-fbl) and --generateBaseline (-gbl) options cannot both be set. Either a baseline is being generated or being used to filter but not both."); + } + + // Verify that the repoRoot exists + if (!Directory.Exists(repoRoot)) + { + throw new ArgumentException($"The repository root '{repoRoot}' is not a valid directory. Please ensure the --repoRoot is set to the root of the repository."); + } + // Verify that the CODEOWNERS file exists in the .github subdirectory of the repository root + string codeownersFileFullPath = Path.Combine(repoRoot, ".github", "CODEOWNERS"); + if (!File.Exists(codeownersFileFullPath)) + { + throw new ArgumentException($"CODEOWNERS file {codeownersFileFullPath} does not exist. Please ensure the --repoRoot is set to the root of the repository and the CODEOWNERS file exists in the .github subdirectory."); + } + // Verify that label data exists for the repository + RepoLabelDataUtils repoLabelData = new RepoLabelDataUtils(repoLabelBlobStorageUri, repoName); + if (!repoLabelData.RepoLabelDataExists()) + { + throw new ArgumentException($"The repository label data for {repoName} does not exist. Should this be running in this repository?"); + } + + string codeownersBaselineFile = Path.Combine(repoRoot, ".github", BaselineConstants.BaselineErrorFile); + bool codeownersBaselineFileExists = false; + // If the baseline is to be used, verify that it exists. + if (filterBaselineErrors) + { + if (File.Exists(codeownersBaselineFile)) + { + codeownersBaselineFileExists = true; + } + else + { + Console.WriteLine($"The CODEOWNERS baseline error file, {codeownersBaselineFile}, file for {repoName} does not exist. No filtering will be done for errors."); + } + } + + DirectoryUtils directoryUtils = new DirectoryUtils(repoRoot); + OwnerDataUtils ownerData = new OwnerDataUtils(teamUserBlobStorageUri, userOrgVisibilityBlobStorageUri); + + var errors = CodeownersUtils.Verification.CodeownersLinter.LintCodeownersFile(directoryUtils, + ownerData, + repoLabelData, + codeownersFileFullPath); + + // Regenerate the baseline file if that option was selected + if (generateBaseline) + { + BaselineUtils baselineUtils = new BaselineUtils(codeownersBaselineFile); + baselineUtils.GenerateBaseline(errors); + } + + // If the baseline is being used to filter out known errors, set the list + // of errors to the filtered list. + if (filterBaselineErrors) + { + // Can only filter if the filter file exists, if it doesn't then there's nothing to filter + // and all encountered errors will be output. Also, if the file doesn't exist that's reported + // above and doesn't need to be reported here. + if (codeownersBaselineFileExists) + { + BaselineUtils baselineUtils = new BaselineUtils(codeownersBaselineFile); + errors = baselineUtils.FilterErrorsUsingBaseline(errors); + } + } + + int returnCode = 0; + // If there are errors, ensure the returnCode is non-zero and output the errors. + if (errors.Count > 0) + { + returnCode = 1; + + // Output the errors sorted ascending by line number and by type. If there's a block + // error with the same starting line number as a single line error, the block error + // should be output first. + var errorsByLineAndType = errors.OrderBy(e => e.LineNumber).ThenBy(e => e.GetType().Name); + + foreach (var error in errorsByLineAndType) + { + Console.WriteLine(error + Environment.NewLine); + } + } + return returnCode; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj new file mode 100644 index 00000000000..b732ed8ca9a --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj @@ -0,0 +1,13 @@ + + + + net6.0 + disable + disable + + + + + + + diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/RepoLabelCache.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/RepoLabelCache.cs new file mode 100644 index 00000000000..aecc0993fed --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/RepoLabelCache.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Caches +{ + /// + /// Holder for the repo label dictionary. Lazy init, it doesn't pull the data from the URI until first use. + /// The GetRepoLabelData is interesting because the repo/label data is a dictionary of hashsets but, both + /// the dictionary and all of the hashsets need to be case insensitive because GitHub is case insensitive + /// but case preserving when it comes to labels. This means that MyLabel, mylabel and mYlAbEl are effectively + /// the same to GitHub and, let's face it, the CODEOWNERS files are all over the place with exact vs non-exact + /// casing. + /// + public class RepoLabelCache + { + private string RepoLabelBlobStorageURI { get; set; } = DefaultStorageConstants.RepoLabelBlobStorageURI; + private Dictionary> _repoLabelDict = null; + + public Dictionary> RepoLabelDict + { + get + { + if (_repoLabelDict == null) + { + _repoLabelDict = GetRepoLabelData(); + } + return _repoLabelDict; + } + set + { + _repoLabelDict = value; + } + } + + public RepoLabelCache(string repoLabelBlobStorageURI) + { + if (!string.IsNullOrWhiteSpace(repoLabelBlobStorageURI)) + { + RepoLabelBlobStorageURI = repoLabelBlobStorageURI; + } + } + + private Dictionary> GetRepoLabelData() + { + if (null == _repoLabelDict) + { + string rawJson = FileHelpers.GetFileOrUrlContents(RepoLabelBlobStorageURI); + var tempDict = JsonSerializer.Deserialize>>(rawJson); + // The StringComparer needs to be set in order to do an case insensitive lookup for both + // the repository (the dictionary key) and for the HashSet of labels. + _repoLabelDict = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); + foreach (KeyValuePair> entry in tempDict) + { + _repoLabelDict.Add(entry.Key, new HashSet(entry.Value, StringComparer.InvariantCultureIgnoreCase)); + } + } + return _repoLabelDict; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/TeamUserCache.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/TeamUserCache.cs new file mode 100644 index 00000000000..8585bd49800 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/TeamUserCache.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Caches +{ + /// + /// Holds the team/user information which is used for both verification and team expansion during parsing. Note that + /// Teams are case insensitive but case preserving which means the dictionary needs to be able to case insensitive lookups. + /// + public class TeamUserCache + { + private string TeamUserStorageURI { get; set; } = DefaultStorageConstants.TeamUserBlobUri; + private Dictionary> _teamUserDict = null; + public Dictionary> TeamUserDict + { + get + { + if (_teamUserDict == null) + { + _teamUserDict = GetTeamUserData(); + } + return _teamUserDict; + } + set + { + _teamUserDict = value; + } + } + + public TeamUserCache(string teamUserStorageURI) + { + if (!string.IsNullOrWhiteSpace(teamUserStorageURI)) + { + TeamUserStorageURI = teamUserStorageURI; + } + } + + private Dictionary> GetTeamUserData() + { + if (null == _teamUserDict) + { + string rawJson = FileHelpers.GetFileOrUrlContents(TeamUserStorageURI); + var list = JsonSerializer.Deserialize>>>(rawJson); + if (null != list) + { + // The StringComparer needs to be set in order to do an case insensitive lookup. GitHub's teams + // and users are case insensitive but case preserving. This means that a team can be @Azure/SomeTeam + // but, in a CODEOWNERS file, it can be @azure/someteam and queries to get users for the team need to + // succeed regardless of casing. + return list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value, StringComparer.InvariantCultureIgnoreCase); + } + Console.WriteLine($"Error! Unable to deserialize json team/user data from {TeamUserStorageURI}. rawJson={rawJson}"); + return new Dictionary>(); + } + return _teamUserDict; + } + + public List GetUsersForTeam(string teamName) + { + // The teamName in the codeowners file should be in the form /. + // The dictionary's team names do not contain the org so the org needs to + // be stripped off. Handle the case where the teamName passed in does and + // does not begin with @org/ + string teamWithoutOrg = teamName.Trim(); + if (teamWithoutOrg.Contains(SeparatorConstants.Team)) + { + teamWithoutOrg = teamWithoutOrg.Split(SeparatorConstants.Team, StringSplitOptions.TrimEntries)[1]; + } + if (TeamUserDict != null) + { + if (TeamUserDict.ContainsKey(teamWithoutOrg)) + { + return TeamUserDict[teamWithoutOrg]; + } + Console.WriteLine($"Warning: TeamUserDictionary did not contain a team entry for {teamWithoutOrg}"); + } + return new List(); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/UserOrgVisibilityCache.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/UserOrgVisibilityCache.cs new file mode 100644 index 00000000000..3916f48afd9 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Caches/UserOrgVisibilityCache.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Caches +{ + /// + /// Holder for the user/org visibility data. The users are the all inclusive list of user under azure-sdk-write meaning + /// that it's effectively a deduped list of users from azure-sdk-write team and its child teams. Owners are case + /// insensitive but case preserving so this needs to be able to do case insensitive lookups. + /// + public class UserOrgVisibilityCache + { + private string UserOrgVisibilityBlobStorageURI { get; set; } = DefaultStorageConstants.UserOrgVisibilityBlobStorageURI; + private Dictionary _userOrgDict = null; + + public Dictionary UserOrgVisibilityDict + { + get + { + if (_userOrgDict == null) + { + _userOrgDict = GetUserOrgVisibilityData(); + } + return _userOrgDict; + } + set + { + _userOrgDict = value; + } + } + + public UserOrgVisibilityCache(string userOrgVisibilityBlobStorageUri) + { + if (!string.IsNullOrWhiteSpace(userOrgVisibilityBlobStorageUri)) + { + UserOrgVisibilityBlobStorageURI = userOrgVisibilityBlobStorageUri; + } + } + + private Dictionary GetUserOrgVisibilityData() + { + if (null == _userOrgDict) + { + string rawJson = FileHelpers.GetFileOrUrlContents(UserOrgVisibilityBlobStorageURI); + var tempDict = JsonSerializer.Deserialize>(rawJson); + // The StringComparer needs to be set in order to do an case insensitive lookup. GitHub's teams + // and users are case insensitive but case preserving. This means that a user's login can be + // SomeUser but, in a CODEOWNERS file, can be @someuser and it's the same user. + _userOrgDict = new Dictionary(tempDict, StringComparer.InvariantCultureIgnoreCase); + } + return _userOrgDict; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/BaselineConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/BaselineConstants.cs new file mode 100644 index 00000000000..892c6ee7fb6 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/BaselineConstants.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// Codeowners baseline error file name + /// + public class BaselineConstants + { + public const string BaselineErrorFile = "CODEOWNERS_baseline_errors.txt"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/DefaultStorageConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/DefaultStorageConstants.cs new file mode 100644 index 00000000000..48d4783c331 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/DefaultStorageConstants.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// Default constants for the storage URIs if they're not passed in. + /// + public class DefaultStorageConstants + { + public const string RepoLabelBlobStorageURI = "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/repository-labels-blob"; + public const string TeamUserBlobUri = "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob"; + public const string UserOrgVisibilityBlobStorageURI = "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/user-org-visibility-blob"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/ErrorMessageConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/ErrorMessageConstants.cs new file mode 100644 index 00000000000..dc574deafa5 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/ErrorMessageConstants.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// Class to store error message constant strings. Anything that's a partial needs to have something + /// prepended to it. For example: The NoWritePermissionPartial is an Owner error and requires the + /// owner to be prepended whereas NeedsToEndWithSourceOwnerPartial is a moniker error and requires + /// the moniker prepended + /// + public class ErrorMessageConstants + { + // Invalid CODEOWNERS path or file. This can happen if the repoRoot argument is incorrect or + // if someone opted to try and sparse checkout the pipeline where this is running (it + // cannot do all of the necessary checks in sparsely checked out repositor)y. + public const string PathOrFileNotExistInRepoPartial = " path or file does not exist in repository."; + + // Owner errors + public const string InvalidTeamPartial = " is an invalid team. Ensure the team exists and has write permissions."; + public const string InvalidUserPartial = " is an invalid user. Ensure the user exists, is public member of Azure and has write permissions."; + public const string MalformedTeamEntryPartial = " is a malformed team entry and should start with '@Azure/'."; + public const string NoOwnersDefined = "There are no owners defined for CODEOWNERS entry."; + public const string NotAPublicMemberOfAzurePartial = " is not a public member of Azure."; + + // Label errors + public const string InvalidRepositoryLabelPartial = " is not a valid label for this repository."; + public const string MissingLabelForMoniker = "Moniker requires a label entry."; + public const string ServiceAttentionIsNotAValidPRLabel = $"{LabelConstants.ServiceAttention} is not a valid label for {MonikerConstants.PRLabel}"; + public const string ServiceLabelMustContainAServiceLabel = $"{MonikerConstants.ServiceLabel} is must contain a valid label, not just the {LabelConstants.ServiceAttention} label."; + + // Invalid CODEOWNERS source path messages + public const string MustStartWithASlashPartial = $" does not start with a '/'"; + public const string GlobCannotEndWithSingleSlashTwoAsterisksPartial = $" ends with an unsupported sequence '{GlobConstants.SingleSlashTwoAsterisks}' and will not match. Replace it with '{GlobConstants.SingleSlash}'"; + public const string GlobCannotEndWithSingleSlashTwoAsterisksSingleSlashPartial = $" ends with an unsupported sequence '{GlobConstants.SingleSlashTwoAsterisksSingleSlash}' and will not match. Replace it with '{GlobConstants.SingleSlash}'"; + public const string GlobCannotEndInWildCardPartial = $" ends in a wildcard '{GlobConstants.SingleAsterisk}'. For directories use '{GlobConstants.SingleAsterisk}{GlobConstants.SingleSlash}' to match files in multiple directories starting with. For files match the file extension, '{GlobConstants.SingleAsterisk}.md' or all files in a single directory '{GlobConstants.SingleSlash}{GlobConstants.SingleAsterisk}'"; + + // Invalid CODEOWNERS Glob Character Error messages + public const string ContainsEscapedPoundPartial = $" contains {GlobConstants.EscapedPound}. Escaping a pattern starting with # using \\ so it is treated as a pattern and not a comment will not work in CODEOWNERS"; + public const string ContainsNegationPartial = $" contains {GlobConstants.ExclamationMark}. Using {GlobConstants.ExclamationMark} to negate a pattern will not work in CODEOWNERS"; + public const string ContainsQuestionMarkPartial = $" contains {GlobConstants.QuestionMark}. Please use {GlobConstants.SingleAsterisk} instead."; + public const string ContainsRangePartial = $" contains {GlobConstants.LeftBracket} and/or {GlobConstants.RightBracket}. Character ranges will not work in CODEOWNERS"; + + // Invalid CODEOWNERS Glob Patterns Error messages + public const string PathIsSingleSlash = $"Path is '{GlobConstants.SingleSlash}' and will never match anything. Use '{GlobConstants.SingleSlashTwoAsterisks}' instead."; + public const string PathIsSingleSlashTwoAsterisksSingleSlash = $"Path is '{GlobConstants.SingleSlashTwoAsterisksSingleSlash}' which is invalid. Use '{GlobConstants.SingleSlashTwoAsterisks}' instead."; + + // Codeowners glob is syntactically but doesn't match anything + public const string GlobHasNoMatchesInRepoPartial = " glob does not have any matches in repository."; + + // Block formatting errors. These errors are specifically around validation blocks. For example, the AzureSdkOwner moniker needs + // to part of a block that ends in a source path/owners line so it's known what they own. + public const string ServiceLabelNeedsOwners = $"{MonikerConstants.ServiceLabel} needs to be followed by, {MonikerConstants.MissingFolder} or {MonikerConstants.ServiceOwners} with owners, or a source path/owner line."; + public const string ServiceLabelHasTooManyOwners = $"{MonikerConstants.ServiceLabel} cannot be part of a block with, {MonikerConstants.MissingFolder} or {MonikerConstants.ServiceOwners}, and a source path/owner line."; + public const string ServiceLabelHasTooManyOwnerMonikers = $"{MonikerConstants.ServiceLabel} cannot be part of a block with both {MonikerConstants.ServiceOwners} and {MonikerConstants.MissingFolder}."; + public const string MissingServiceLabelPartial = $" needs to be part of a block with a {MonikerConstants.ServiceLabel} entry."; + public const string NeedsToEndWithSourceOwnerPartial = " needs to be part of a block that ends in a source path/owner line."; + // Duplicate Moniker error + public const string DuplicateMonikerInBlockPartial = " already exists in the block. A moniker cannot exist more than once in a block."; + public const string AzureSdkOwnersMustBeWithServiceLabel = $"{MonikerConstants.AzureSdkOwners} must be part of a block that contains a {MonikerConstants.ServiceLabel} entry."; + public const string ServiceOwnersMustBeWithServiceLabel = $"{MonikerConstants.ServiceOwners} must be part of a block that contains a {MonikerConstants.ServiceLabel} entry."; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/GlobConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/GlobConstants.cs new file mode 100644 index 00000000000..2dc2a405715 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/GlobConstants.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// Contains the patterns and characters that are either special or invalid for a CODEOWNERS file as defined by GitHub documentation. + /// https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax + /// There is one additional things that is disallowed and that is the question mark operator. This was disallowed by the original + /// CodeownersParser. + /// + public class GlobConstants + { + // Escaping a pattern starting with # using \ so it is treated as a pattern and not a comment + public const string EscapedPound = @"\#"; + // While the following are characters, they need to be string to be used in the ErrorMessageConstants + // Using ! to negate a pattern + public const string ExclamationMark = "!"; + // Using [ ] to define a character range + public const string LeftBracket = "["; + public const string RightBracket = "]"; + public const string QuestionMark = "?"; + + + // The following are all specical characters/glob patterns that need to be checked when + // looking for unsupported character sequences. + + // Wildcarding file/directory names with a "*", outside of "/*" is invalid. The globber + // won't be able to deal with them correctly. For directories, instead of /dirPartial*, + // /dirPartial*/ should be used. + public const string SingleAsterisk = "*"; + // A Single slash "/" is unsupported by GitHub + public const string SingleSlash = "/"; + // Two asterisks is legal if surrounded by single slashes but, if not, it's otherwise + // equivalent to a single star which is what should be used to avoid confusion. + public const string TwoAsterisks = "**"; + // The suffix of "/**" is not supported because it is equivalent to "/". For example, + // "/foo/**" is equivalent to "/foo/". One exception to this rule is if the entire + // path is "/**". The reason being is that GitHub doesn't match "/" to anything, + // if it is the entire path but, instead, expects "/**". + public const string SingleSlashTwoAsterisks = "/**"; + // "/**/" as a suffix is equivalent to the suffix "/**" which is equivalent to the suffix "/" + // which is what should be being used to avoid confusion + public const string SingleSlashTwoAsterisksSingleSlash = "/**/"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/LabelConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/LabelConstants.cs new file mode 100644 index 00000000000..568dafe6dee --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/LabelConstants.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// The only special label that's currently needed for processing. + /// + public class LabelConstants + { + public const string ServiceAttention = "Service Attention"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/MonikerConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/MonikerConstants.cs new file mode 100644 index 00000000000..f8d7d6ff51e --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/MonikerConstants.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + /// + /// The allowed Monikers in CODEOWNERS files. + /// + public class MonikerConstants + { + // Note: AzureSdkOwners and ServiceOwner aren't implemented yet. They're + // opportunistically being added below with the move of the moniker constants. + // https://github.com/Azure/azure-sdk-tools/issues/5945 + public const string AzureSdkOwners = "AzureSdkOwners"; + public const string MissingFolder = "//"; + public const string PRLabel = "PRLabel"; + public const string ServiceLabel = "ServiceLabel"; + public const string ServiceOwners = "ServiceOwners"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/OrgConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/OrgConstants.cs new file mode 100644 index 00000000000..3abcc04022a --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/OrgConstants.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + public class OrgConstants + { + // Azure is both the Org and the owner of the various repositories + public const string Azure = "Azure"; + // Everyone with write permission in the Azure org either needs to be a direct user + // in the azure-sdk-write team or a user in a child team of azure-sdk-write. + public const string AzureSdkWriteTeam = "azure-sdk-write"; + public const string AzureOrgTeamConstant = $"{Azure}/"; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/SeparatorConstants.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/SeparatorConstants.cs new file mode 100644 index 00000000000..0fbcd2859ee --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Constants/SeparatorConstants.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Constants +{ + // The various separators used in CODEOWNERS + public class SeparatorConstants + { + public const char Label = '%'; + public const char Owner = '@'; + public const char Team = '/'; + public const char Colon = ':'; + public const char Comment = '#'; + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BaseError.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BaseError.cs new file mode 100644 index 00000000000..6c3c1f50414 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BaseError.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Errors +{ + /// + /// The base error class, used by SingleLineError and BlockFormatting, with the common methods and + /// members. Note that the ToString must be overridden by the concrete classes. + /// + public abstract class BaseError + { + protected const string Indent = " "; + protected const string IndentWithDash = $"{Indent}-"; + + public List Errors { get; private set; } + + // For single line errors this will be the line number with the error + // but for multi-line errors this is the start line. This is done this + // way so errors can be processed independenly and sorted by line/start line. + public int LineNumber { get; private set; } + + public BaseError(int lineNumber, string error) + { + LineNumber = lineNumber; + List errors = new List + { + error + }; + Errors = errors; + } + public BaseError(int lineNumber, List errors) + { + LineNumber = lineNumber; + Errors = errors; + } + + // Create a single error string separated by NewLine characters with indents and dashes + public string ErrorString + { + get + { + return string.Join(Environment.NewLine, Errors.Select(s => $"{IndentWithDash}{s}")); + } + } + public abstract override string ToString(); + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BlockFormattingError.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BlockFormattingError.cs new file mode 100644 index 00000000000..eabce8b79b6 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/BlockFormattingError.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Errors +{ + /// + /// BlockFormattingError is used to report errors for a source block that spans multiple lines. It's used to display the + /// entire block of from the CODEOWNERS file. For example, if a block contains ServiceLabel and a PRLabel but + /// doesn't end in a source path/owners line, the error would display the start/end line numbers with the actual block + /// in between. For example: + /// Source block error. + /// Source block start: 120 + /// actual source line 120 + /// actual source line 121 + /// actual source line 122 + /// Source block end: 122 + /// - first error + /// - second error + /// + public class BlockFormattingError: BaseError + { + public int EndLineNumber { get; private set; } + public List SourceBlock { get; private set; } + public BlockFormattingError(int startLine, int endLine, List sourceBlock, string error) : base(startLine, error) + { + EndLineNumber = endLine; + SourceBlock = sourceBlock; + } + public BlockFormattingError(int startLine, int endLine, List sourceBlock, List errors) : base(startLine, errors) + { + EndLineNumber = endLine; + SourceBlock = sourceBlock; + } + + public string SourceBlockString + { + get + { + return string.Join(Environment.NewLine, SourceBlock.Select(s => $"{Indent}{s}")); + } + } + public override string ToString() + { + var returnString = string.Join( + Environment.NewLine, + $"Source block error.", + $"Source Block Start: {LineNumber}", + $"{SourceBlockString}", + $"Source Block End: {EndLineNumber}", + ErrorString + ); + return returnString; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/SingleLineError.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/SingleLineError.cs new file mode 100644 index 00000000000..de09b7c6750 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Errors/SingleLineError.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace Azure.Sdk.Tools.CodeownersUtils.Errors +{ + /// + /// SingleLineError is the error class to hold any/all issues with a single line being parsed. + /// For example, a source/path @owner1...@ownerX line could have different issues with several + /// owners. The error string would look like: + /// Error on line 500. + /// Source line: source/path @owner1...@ownerX + /// - first error + /// - second error + /// + public class SingleLineError : BaseError + { + public string Source { get; private set; } + + public SingleLineError(int lineNumber, string sourceText, string error) : base(lineNumber, error) + { + Source = sourceText; + } + public SingleLineError(int lineNumber, string sourceText, List errors) : base(lineNumber, errors) + { + Source = sourceText; + } + public override string ToString() + { + var returnString = string.Join( + Environment.NewLine, + $"Error(s) on line {LineNumber}", + $"Source Line: {Source}", + ErrorString + ); + return returnString; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersEntry.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersEntry.cs new file mode 100644 index 00000000000..9584502da34 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersEntry.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Caches; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Parsing +{ + /// + /// The entry for CODEOWNERS has one othe following structures: + /// + /// A plain source path/owner entry + /// + /// + /// path @owner @owner + /// + /// + /// A source path/owner entry with a PRLabel + /// + /// + /// # PRLabel: %Label + /// path @owner @owner + /// + /// + /// A source path/owner entry PRLabel and ServiceLabel entries. In this case Service Owners + /// will be same as source owners. + /// + /// + /// # PRLabel: %Label + /// # ServiceLabel: %Label + /// path @owner @owner + /// + /// + /// A source path/owner entry PRLabel, ServiceLabel entries + /// + /// + /// # PRLabel: %Label1 %Label2 + /// # ServiceLabel: %Label3 %Label4 + /// path @owner @owner + /// + /// + /// ServiceLabel and ServiceOwners entry OR /<NotInRepo>/ (one or the other, not both). ServiceOwners is preferred going forward. + /// + /// + /// # ServiceLabel: %Label3 %Label4 + /// # ServiceOwners: @owner @owner + /// or, the old style + /// # ServiceLabel: %Label3 %Label4 + /// # /<NotInRepo>/: @owner @owner + /// + /// + public class CodeownersEntry + { + public string PathExpression { get; set; } = ""; + + public bool ContainsWildcard => PathExpression.Contains('*'); + + public List SourceOwners { get; set; } = new List(); + + // PRLabels are tied to a source path/owner line + // In theory, there should only have be one PR Label. + public List PRLabels { get; set; } = new List(); + + // ServiceLabels are tied to either, ServiceOwners or // (MissingLabel moniker), or + // the owners from a source path/owners line (meaning that the owners for the source are also + // the service owners) + // In theory, there should only have ever be one ServiceLabel, aside from the ServiceAttention label + public List ServiceLabels { get; set; } = new List(); + + public List ServiceOwners { get; set; } = new List(); + + // AzureSdkOwners are directly tied to the source path. If the AzureSdkOwners are defined + // on the same line as the moniker, it'll use those, if it's empty it'll use the owners from + // the source path/owner line. + public List AzureSdkOwners { get; set; } = new List(); + + public bool IsValid => !string.IsNullOrWhiteSpace(PathExpression); + + public CodeownersEntry() + { + } + public override string ToString() + { + return string.Join( + Environment.NewLine, + $"PathExpression:{PathExpression}, HasWildcard:{ContainsWildcard}", + $"SourceOwners:{string.Join(", ", SourceOwners)}", + $"PRLabels:{string.Join(", ", PRLabels)}", + $"ServiceLabels:{string.Join(", ", ServiceLabels)}", + $"ServiceOwners:{string.Join(", ", ServiceOwners)}", + $"AzureSdkOwners:{string.Join(", ", AzureSdkOwners)}" + ); + } + + /// + /// Remove all code owners which are not github alias. + /// Even with team expansion there can still be teams in lists. This + /// can happen if the team is not a child team under azure-sdk-write, which + /// are the only teams expanded. + /// + public void ExcludeNonUserAliases() + { + SourceOwners.RemoveAll(r => ParsingUtils.IsGitHubTeam(r)); + ServiceOwners.RemoveAll(r => ParsingUtils.IsGitHubTeam(r)); + AzureSdkOwners.RemoveAll(r => ParsingUtils.IsGitHubTeam(r)); + } + + protected bool Equals(CodeownersEntry other) + => PathExpression == other.PathExpression + && SourceOwners.SequenceEqual(other.SourceOwners) + && ServiceOwners.SequenceEqual(other.ServiceOwners) + && AzureSdkOwners.SequenceEqual(other.AzureSdkOwners) + && PRLabels.SequenceEqual(other.PRLabels) + && ServiceLabels.SequenceEqual(other.ServiceLabels); + + public override bool Equals(object obj) + { + // @formatter:off + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((CodeownersEntry)obj); + // @formatter:on + } + + /// + /// Implementation of GetHashCode that properly hashes collections. + /// Implementation based on + /// https://stackoverflow.com/a/10567544/986533 + /// + /// This implementation is candidate to be moved to: + /// https://github.com/Azure/azure-sdk-tools/issues/5281 + /// + public override int GetHashCode() + { + int hashCode = 0; + // ReSharper disable NonReadonlyMemberInGetHashCode + hashCode = AddHashCodeForObject(hashCode, PathExpression); + hashCode = AddHashCodeForEnumerable(hashCode, SourceOwners); + hashCode = AddHashCodeForEnumerable(hashCode, ServiceOwners); + hashCode = AddHashCodeForEnumerable(hashCode, AzureSdkOwners); + hashCode = AddHashCodeForEnumerable(hashCode, PRLabels); + hashCode = AddHashCodeForEnumerable(hashCode, ServiceLabels); + // ReSharper restore NonReadonlyMemberInGetHashCode + return hashCode; + + // ReSharper disable once VariableHidesOuterVariable + int AddHashCodeForEnumerable(int hashCode, IEnumerable enumerable) + { + foreach (var item in enumerable) + { + hashCode = AddHashCodeForObject(hashCode, item); + } + return hashCode; + } + + int AddHashCodeForObject(int hc, object item) + { + // Based on https://stackoverflow.com/a/10567544/986533 + hc ^= item.GetHashCode(); + hc = (hc << 7) | + (hc >> (32 - 7)); // rotate hashCode to the left to swipe over all bits + return hc; + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersParser.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersParser.cs new file mode 100644 index 00000000000..b1bbbde92a4 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersParser.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Utils; +using Azure.Sdk.Tools.CodeownersUtils.Verification; + +namespace Azure.Sdk.Tools.CodeownersUtils.Parsing +{ + /// + /// This is the main entry point for Codeowners parsing. In the old parser this is the equivalent of CodeownersFile. + /// This just contains the static methods for parsing the CODEOWNERS file and for retrieving matching CODEOWNS entries. + /// + public class CodeownersParser + { + /// + /// Load the CODEOWNERS content from the file or URL and parse the entries. + /// + /// + /// + /// + public static List ParseCodeownersFile(string codeownersFilePathOrUrl, + string teamStorageURI = null) + { + List codeownersFile = FileHelpers.LoadFileAsStringList(codeownersFilePathOrUrl); + return ParseCodeownersEntries(codeownersFile, teamStorageURI); + } + + /// + /// Given a CODEOWNERS file as a List>string<, parse the Codeowners entries. + /// Codeowners file as a List>string< + /// The URI of the team storage data if being overridden. + /// List>CodeownersEntry< + public static List ParseCodeownersEntries(List codeownersFile, + string teamStorageURI = null) + { + OwnerDataUtils ownerDataUtils = new OwnerDataUtils(teamStorageURI); + List codeownersEntries = new List(); + + // Start parsing the codeowners file, a block at a time. + // A block can be one of the following: + // 1. A single source path/owner line + // 2. One or more monikers that either ends in source path/owner line or a blank line, depending + // on the moniker. + for (int currentLineNum = 0; currentLineNum < codeownersFile.Count; currentLineNum++) + { + string line = codeownersFile[currentLineNum]; + if (ParsingUtils.IsMonikerOrSourceLine(line)) + { + // A block can be a single line, if it's a source path/owners line, or if the block starts + // with a moniker, it'll be multi-line + int blockEnd = ParsingUtils.FindBlockEnd(currentLineNum, codeownersFile); + List errors = new List(); + CodeownersLinter.VerifyBlock(errors, + currentLineNum, + blockEnd, + codeownersFile); + if (errors.Count > 0) + { + // There should only be a single block error here. + foreach(BaseError error in errors) + { + Console.Error.WriteLine($"Block error encountered while parsing, entry will be skipped.\n{error}"); + } + } + else + { + codeownersEntries.Add(ParseCodeownersEntryFromBlock(ownerDataUtils, currentLineNum, blockEnd, codeownersFile)); + } + // After processing the block, set the current line to the end line which will get + // incremented and continue the processing the line after the block + currentLineNum = blockEnd; + } + } + return codeownersEntries; + } + + /// + /// Given a + /// + /// OwnerDataUtils, required for team expansion only. + /// Starting line number of the block. + /// Ending line number of the block. + /// Codeowners file as a List>string< + /// CodeownersEntry for the parsed block. + /// Thrown if a moniker encountered isn't in the switch statement. + public static CodeownersEntry ParseCodeownersEntryFromBlock(OwnerDataUtils ownerDataUtils, + int startBlockLineNumber, + int endBlockLineNumber, + List codeownersFile) + { + CodeownersEntry codeownersEntry = new CodeownersEntry(); + // If the block ends with a source path/owner line then any owner moniker line that are empty + // will be set to the same list as the source owners. + bool endsWithSourceOwnerLine = ParsingUtils.IsSourcePathOwnerLine(codeownersFile[endBlockLineNumber]); + // These are used in the case where the AzureSdkOwners and/or ServiceOwners are empty and part of a + // block that ends in a source path/owners line. This means that the either or both monikers will have + // their owner lists set to the same owners as source owners. + bool emptyAzureSdkOwners = false; + bool hasServiceLabel = false; + + for (int blockLine = startBlockLineNumber; blockLine <= endBlockLineNumber; blockLine++) + { + string line = codeownersFile[blockLine]; + bool isSourcePathOwnerLine = ParsingUtils.IsSourcePathOwnerLine(line); + if (isSourcePathOwnerLine) + { + codeownersEntry.SourceOwners = ParsingUtils.ParseOwnersFromLine(ownerDataUtils, + line, + true /*expand teams when parsing*/); + // So it's clear why this is here: + // The original parser left the PathExpression empty if there were no source owners for a given path + // in order to prevent matches against a PathExpression with no source owners. The same needs to be + // done here for compat reasons. + if (codeownersEntry.SourceOwners.Count != 0) + { + codeownersEntry.PathExpression = ParsingUtils.ParseSourcePathFromLine(line); + } + } + else + { + string moniker = MonikerUtils.ParseMonikerFromLine(line); + // A block can contain comments which is why a line in a block wouldn't have a moniker + if (moniker == null) + { + continue; + } + switch (moniker) + { + case MonikerConstants.AzureSdkOwners: + { + codeownersEntry.AzureSdkOwners = ParsingUtils.ParseOwnersFromLine(ownerDataUtils, + line, + true /*expand teams when parsing*/); + if (codeownersEntry.AzureSdkOwners.Count == 0) + { + emptyAzureSdkOwners = true; + } + break; + } + case MonikerConstants.PRLabel: + { + codeownersEntry.PRLabels = ParsingUtils.ParseLabelsFromLine(line); + break; + } + case MonikerConstants.ServiceLabel: + { + codeownersEntry.ServiceLabels = ParsingUtils.ParseLabelsFromLine(line); + hasServiceLabel = true; + break; + } + // ServiceOwners and // both map to service owners. + case MonikerConstants.MissingFolder: + case MonikerConstants.ServiceOwners: + { + codeownersEntry.ServiceOwners = ParsingUtils.ParseOwnersFromLine(ownerDataUtils, + line, + true /*expand teams when parsing*/); + break; + } + default: + // This shouldn't get here unless someone adds a new moniker and forgets to add it to the switch statement + throw new ArgumentException($"Unexpected moniker '{moniker}' found on line {blockLine+1}\nLine={line}"); + + } + } + } + + // Take care of the case where an empty owners moniker, in a block that ends + // in a source path/owners moniker, uses the source owners as its owners. + if (endsWithSourceOwnerLine) + { + // If the AzureSdkOwners moniker had no owners defined, the AzureSdkOwners are + // the same as the SourceOwners + if (emptyAzureSdkOwners) + { + codeownersEntry.AzureSdkOwners = codeownersEntry.SourceOwners; + } + // If there was a ServiceLabel and the block ended in a source path/owners line, the + // ServiceOwners are the same as the SourceOWners + if (hasServiceLabel && endsWithSourceOwnerLine) + { + codeownersEntry.ServiceOwners = codeownersEntry.SourceOwners; + } + } + return codeownersEntry; + } + + /// + /// Given a list of Codeowners entries and a target patch, return the entry that matches the target path. Note: + /// the function processes the list in reverse order, in other words it'll return the match in the list which + /// mirrors the way GitHub would match from the CODEOWNERS file; The last entry in the CODEOWNERS file that matches + /// is the one that "wins". + /// + /// The path to check. + /// List>CodeownersEntry< parsed the CODEOWNERS file in the repository that the path belongs to. + /// The CodeownersEntry that matches the target path or an empty CodeownersEntry if there is no match + public static CodeownersEntry GetMatchingCodeownersEntry(string targetPath, List codeownersEntries) + { + Debug.Assert(!string.IsNullOrWhiteSpace(targetPath)); + CodeownersEntry matchedEntry = codeownersEntries + .Where(entry => DirectoryUtils.PathExpressionMatchesTargetPath(entry.PathExpression, targetPath)) + // Entries listed in CODEOWNERS file below take precedence, hence we read the file from the bottom up. + // By convention, entries in CODEOWNERS should be sorted top-down in the order of: + // - 'RepoPath', + // - 'ServicePath' + // - and then 'PackagePath'. + // However, due to lack of validation, as of 12/29/2022 this is not always the case. + .Reverse() + // Return the first element found or a new, empty CodeownersEntry if nothing matched + .FirstOrDefault(new CodeownersEntry()); + return matchedEntry; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/BaselineUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/BaselineUtils.cs new file mode 100644 index 00000000000..20558352de4 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/BaselineUtils.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Errors; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// Utilities for processing the CODEOWNERS baseline error file as well as processing errors + /// based upon the contents of the file. The baseline error file is simply a text file that + /// contains every error, on its own line. The errors have been deduped so any given error + /// will only exist in the file once. + /// + public class BaselineUtils + { + private string _baselineFile = null; + public BaselineUtils() + { + } + public BaselineUtils(string baselineFile) + { + _baselineFile = Path.GetFullPath(baselineFile); + } + + /// + /// Given a list of errors for the repository, generate the baseline file. Note that the baseline only consists of + /// single line errors, in other words errors that are for owners and labels, not blocks. The reason being is that + /// these errors are deduped and block errors are thrown out while parsing. This means that if someone screws up + /// a block and that error ends up in the deduped list, that someone else could add a new block with the same error + /// that wouldn't get flagged. This would result in someone adding an incorrect block which would get tossed out in + /// parsing and possibly prevent processing somewhere else because the entry isn't there. (for example adding labels + /// to a PR based upon files paths wouldn't work if the block that the PRLabel was in was thrown out because it was + /// malformed). + /// + /// The list of errors + /// HashSet<string> containing the unique errors. Used in testing for verification. + public void GenerateBaseline(List errors) + { + HashSet uniqueErrors = new HashSet(); + + // Filter out block errors. + var lineErrors = errors.OfType().ToList(); + + // For each error get the error string and add it to the hash if + // isn't already in there. + foreach (var error in lineErrors) + { + foreach (string errorString in error.Errors) + { + if (!uniqueErrors.Contains(errorString)) + { + uniqueErrors.Add(errorString); + } + } + } + + // The HashSet will contain the unique list of errors and those + // will be written out to the baseline file. If there are no errors + // then this will cause an empty file to be written out. + using (var sw = new StreamWriter(_baselineFile)) + { + foreach (string errorString in uniqueErrors) + { + sw.WriteLine(errorString); + } + } + } + + /// + /// Given a list of errors from parsing the CODEOWNERS file, trim it down to only the + /// errors that don't already exist in the baseline. + /// + /// List<BaseError> from parsing the CODEOWNERS file. + /// List<BaseError> representing the list of errors not in the baseline. + public List FilterErrorsUsingBaseline(List errors) + { + List remainingErrors = new List(); + HashSet uniqueErrors = new HashSet(); + using (var sr = new StreamReader(_baselineFile)) + { + while (sr.ReadLine() is { } line) + { + if (!string.IsNullOrWhiteSpace(line)) + { + uniqueErrors.Add(line); + } + } + } + + // For each error look at the error strings and see if they're already in + // the list of errors, if so remove them. The reason why ToList is being + // used here is to ensure that the original list of errors isn't modified. + foreach (var error in errors.ToList()) + { + // Block formatting errors will not be filtered out + if (error is BlockFormattingError) + { + remainingErrors.Add(error); + continue; + } + + // This might look odd, to use ToList on something that's already a + // list but doing this causes the compiler to create a copy of the list + // so the original can be modified without the "Collection was modified; + // Enumeration operation might not execute." + foreach (var errorString in error.Errors.ToList()) + { + if (uniqueErrors.Contains(errorString)) + { + error.Errors.Remove(errorString); + } + } + // If there are any errors left then add this error to the remainingErrors list + if (error.Errors.Count > 0) + { + remainingErrors.Add(error); + } + } + return remainingErrors; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/DirectoryUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/DirectoryUtils.cs new file mode 100644 index 00000000000..55ba7ab7c42 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/DirectoryUtils.cs @@ -0,0 +1,277 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using System.Xml.XPath; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Caches; +using Microsoft.Extensions.FileSystemGlobbing; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// Contains the utilities used for verification and matching. The instance methods are all used + /// by the linter which requires a repository root to determine if CODEOWNERS path is valid for a + /// given repository. The static methods are also used by the linter but primarily used for matching + /// a CodeownersEntry's path expression to a file passed in. + /// + public class DirectoryUtils + { + private string _repoRoot = null; + + public DirectoryUtils() + { + } + + public DirectoryUtils(string repoRoot) + { + _repoRoot = repoRoot; + } + + /// + /// Verify the source sourcePathEntry entry for the repository. + /// + /// The sourcePathEntry to verify, which may contain a pathExpression + /// Any errorStrings encountered are added to this list. + public virtual void VerifySourcePathEntry(string sourcePathEntry, List errorStrings) + { + string pathWithoutOwners = ParsingUtils.ParseSourcePathFromLine(sourcePathEntry); + // The sourcePathEntry is either a pathExpression sourcePathEntry or a sourcePathEntry relative to the repository root + if (IsGlobFilePath(pathWithoutOwners)) + { + // Ensure the pathExpression pattern is valid for a CODEOWNERS file + if (IsValidCodeownersPathExpression(pathWithoutOwners, errorStrings)) + { + // If the pathExpression pattern is valid, ensure that it has matches in the repository + if (!IsValidGlobPatternForRepo(pathWithoutOwners)) + { + errorStrings.Add($"{pathWithoutOwners}{ErrorMessageConstants.GlobHasNoMatchesInRepoPartial}"); + } + } + } + else + { + // Verify that the sourcePathEntry is valid for the repository + if (!IsValidRepositoryPath(pathWithoutOwners)) + { + errorStrings.Add($"{pathWithoutOwners}{ErrorMessageConstants.PathOrFileNotExistInRepoPartial}"); + } + } + } + + /// + /// Overloaded IsValidCodeownersPathExpression that throws away the error strings. This function is used by + /// the matcher where errors aren't being collected but can still be reported if needed. + /// + /// The pathExpression to check + /// Whether or not errors should be reported during + /// True if the pathExpression doesn't contain invalid CODEOWNERS pathExpression patterns, false otherwise. + public static bool IsValidCodeownersPathExpression(string pathExpression, bool reportErrors = false) + { + List errors = new List(); + bool returnValue = IsValidCodeownersPathExpression(pathExpression, errors); + if (reportErrors) + { + // The original matcher used to write errors to Console.Error, do the same here. + foreach (string error in errors) + { + Console.Error.WriteLine(error); + } + } + return returnValue; + } + + /// + /// Check to see if a pathExpression pattern contains any of the patterns or sequences that are invalid for CODEOWNERS file. + /// + /// The pathExpression to check + /// The list that any discovered errorStrings will be added to. Necessary because there can be multiple errorStrings. + /// True if the pathExpression doesn't contain invalid CODEOWNERS pathExpression patterns, false otherwise. + public static bool IsValidCodeownersPathExpression(string pathExpression, List errorStrings) + { + bool returnValue = true; + + // A path expression can contain all of the invalid characters + if (pathExpression.Contains(GlobConstants.EscapedPound)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.ContainsEscapedPoundPartial}"); + returnValue = false; + } + if (pathExpression.Contains(GlobConstants.ExclamationMark)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.ContainsNegationPartial}"); + returnValue = false; + } + if (pathExpression.Contains(GlobConstants.LeftBracket) || pathExpression.Contains(GlobConstants.RightBracket)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.ContainsRangePartial}"); + returnValue = false; + } + if (pathExpression.Contains(GlobConstants.QuestionMark)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.ContainsQuestionMarkPartial}"); + returnValue = false; + } + + // A path expression needs to start with a single / but that cannot be the entire expression + if (pathExpression == GlobConstants.SingleSlash) + { + errorStrings.Add(ErrorMessageConstants.PathIsSingleSlash); + returnValue = false; + } + else if (pathExpression == GlobConstants.SingleSlashTwoAsterisksSingleSlash) + { + errorStrings.Add($"{ErrorMessageConstants.PathIsSingleSlashTwoAsterisksSingleSlash}"); + returnValue = false; + } + else if (!pathExpression.StartsWith(GlobConstants.SingleSlash)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.MustStartWithASlashPartial}"); + returnValue = false; + } + + // The following glob patterns only need to be checked if the path is a glob path and the glob isn't only "/**" + // or "/**/". Unlike the invalid characters, a path expression won't contain multiple invalid patterns because + // the invalid patterns are all ones the path has to end with. + if (IsGlobFilePath(pathExpression) && pathExpression != GlobConstants.SingleSlashTwoAsterisks) + { + // The suffix of "/**" is not supported because it is equivalent to "/". For example, + // "/foo/**" is equivalent to "/foo/". One exception to this rule is if the entire + // path is "/**". + if (pathExpression.EndsWith(GlobConstants.SingleSlashTwoAsterisks)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.GlobCannotEndWithSingleSlashTwoAsterisksPartial}"); + returnValue = false; + } + // The suffix /**/ is invalid. It's effectively equal to /** which is equivalent to /. + else if (pathExpression.EndsWith(GlobConstants.SingleSlashTwoAsterisksSingleSlash) && + pathExpression != GlobConstants.SingleSlashTwoAsterisksSingleSlash) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.GlobCannotEndWithSingleSlashTwoAsterisksSingleSlashPartial}"); + returnValue = false; + } + // A wildcard pattern ending in * won't work with the file globber. The exception is /* which says match everything + // in that directory. This also means that we can't say all files beginning with foo*. Wildcarding file types *.md + // works just fine. For directories a slash at the end /foo*/ would work as intended. It would match all files under + // directories whose names started with foo, like foobar/myFile or foobar2/myDir/myFile etc. + else if (pathExpression.EndsWith(GlobConstants.SingleAsterisk) && !pathExpression.EndsWith(GlobConstants.SingleSlash + GlobConstants.SingleAsterisk)) + { + errorStrings.Add($"{pathExpression}{ErrorMessageConstants.GlobCannotEndInWildCardPartial}"); + return false; + } + } + return returnValue; + } + + /// + /// Check whether or not the pathExpression pattern has any matches for the repository. This is only + /// called by the linter if the path expression is valid + /// + /// The pathExpression pattern to match. + /// True if the pathExpression pattern has results in the repository, false otherwise. + public List GetRepoFilesForGlob(string pathExpression) + { + // Don't use OrginalIgnoreCase. GitHub is case sensitive for directories + Matcher matcher = new Matcher(StringComparison.Ordinal); + matcher.AddInclude(pathExpression); + + // The unfortunate thing about this pathExpression check is that even with a valid pathExpression, + // there's no real way to know if the pattern has too many matches outside of declaring some arbitrary + // max number and comparing to that + var matches = matcher.GetResultsInFullPath(_repoRoot); + return matches.ToList(); + } + + /// + /// Check whether or not the pathExpression pattern has any matches for the repository. + /// + /// The pathExpression pattern to match. + /// True if the pathExpression pattern has results in the repository, false otherwise. + private bool IsValidGlobPatternForRepo(string glob) + { + var matches = GetRepoFilesForGlob(glob); + if (matches.ToList().Count > 0) + { + return true; + } + return false; + } + + /// + /// Verify that the sourcePathEntry entry exists in the repository if it isn't a glob. + /// + /// the sourcePathEntry to verify + /// True if the sourcePathEntry exists in the repository, false otherwise. + private bool IsValidRepositoryPath(string path) + { + // Path.GetFullPath will normalize the directory separator characters for the platform + // it's running on. If there is a leading '/" on the path it must be trimmed or combine + // will think the path is rooted and will just return the path instead of combining it + // with the repo root. + var fullPath = Path.GetFullPath(Path.Combine(_repoRoot, path.TrimStart('/'))); + + // In CODEOWNERS, the path can be a directory or a file. + return Directory.Exists(fullPath) || File.Exists(fullPath); + } + + /// + /// The '*' is the only character that can denote pathExpression pattern + /// in the used globbing library, per: + /// - https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.filesystemglobbing.matcher?view=dotnet-plat-ext-7.0#remarks + /// - https://learn.microsoft.com/en-us/dotnet/core/extensions/file-globbing#pattern-formats + /// + /// The path to check + /// True if the path is a valid pathExpression path, false otherwise. + public static bool IsGlobFilePath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return false; + } + return path.Contains('*'); + } + + /// + /// Given a pathExpression and a targetPath, usually a partial file relative to the repository's root (ex. + /// sdk/SomeServiceDirectory/SomeFileName) check to see if the pathExpression matches the target path. + /// + /// The path expression to check which may or may not be a glob path. + /// The file or path to check. + /// True if match, false otherwise. + public static bool PathExpressionMatchesTargetPath(string pathExpression, string targetPath) + { + if (!IsValidCodeownersPathExpression(pathExpression)) + { + return false; + } + + // The target path cannot be a wildcard + if (targetPath.Contains('*')) + { + Console.Error.WriteLine( + $"Target path \"{targetPath}\" contains star ('*') which is not supported. " + + "Returning no match without checking for ownership."); + return false; + } + + // The target path can't start with a "/" otherwise the Matcher will try to root it which + // will cause the match to fail. + if (targetPath.StartsWith("/")) + { + targetPath = targetPath.Substring(1); + } + // Don't use OrginalIgnoreCase. GitHub is case sensitive for directories + Matcher matcher = new Matcher(StringComparison.Ordinal); + matcher.AddInclude(pathExpression); + + // The unfortunate thing about this pathExpression check is that even with a valid pathExpression, + // there's no real way to know if the pattern has too many matches outside of declaring some arbitrary + // max number and comparing to that + var matches = matcher.Match(targetPath); + return matches.HasMatches; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/FileHelpers.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/FileHelpers.cs new file mode 100644 index 00000000000..5109598f437 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/FileHelpers.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// Utility class for loading files from paths or URLs. + /// + public static class FileHelpers + { + public static string GetFileOrUrlContents(string fileOrUrl) + { + if (fileOrUrl.StartsWith("https")) + return GetUrlContents(fileOrUrl); + + string fullPath = Path.GetFullPath(fileOrUrl); + if (File.Exists(fullPath)) + return File.ReadAllText(fullPath); + + throw new ArgumentException( + "The path provided is neither local path nor https link. " + + $"Please check your path: '{fileOrUrl}' resolved to '{fullPath}'."); + } + + /// + /// Load a file from the repository into an List<string>. + /// Q) Why is this necessary? + /// A) There are some pieces of metadata that require looking forward to ensure correctness. + /// + /// The file path or URL of the file + /// A List<string> representing the file + public static List LoadFileAsStringList(string fileOrUrl) + { + List codeownersFileAsList = new List(); + // GetFileOrUrlContents will throw + string content = GetFileOrUrlContents(fileOrUrl); + using StringReader sr = new StringReader(content); + while (sr.ReadLine() is { } line) + { + codeownersFileAsList.Add(line); + } + + return codeownersFileAsList; + } + + private static string GetUrlContents(string url) + { + int maxRetries = 3; + int attempts = 1; + int delayTimeInMs = 1000; + using HttpClient client = new HttpClient(); + while (attempts <= maxRetries) + { + try + { + HttpResponseMessage response = client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); + if (response.StatusCode == HttpStatusCode.OK) + { + // This writeline is probably unnecessary but good to have if there are previous attempts that failed + Console.WriteLine($"GetUrlContents for {url} attempt number {attempts} succeeded."); + return response.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + } + else + { + Console.WriteLine($"GetUrlContents attempt number {attempts}. Non-{HttpStatusCode.OK} status code trying to fetch {url}. Status Code = {response.StatusCode}"); + } + } + catch (HttpRequestException httpReqEx) + { + // HttpRequestException means the request failed due to an underlying issue such as network connectivity, + // DNS failure, server certificate validation or timeout. + Console.WriteLine($"GetUrlContents attempt number {attempts}. HttpRequestException trying to fetch {url}. Exception message = {httpReqEx.Message}"); + if (attempts == maxRetries) + { + // At this point the retries have been exhausted, let this rethrow + throw; + } + } + System.Threading.Thread.Sleep(delayTimeInMs); + attempts++; + } + // This will only get hit if the final retry is non-OK status code + throw new FileLoadException($"Unable to fetch {url} after {maxRetries}. See above for status codes for each attempt."); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/MonikerUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/MonikerUtils.cs new file mode 100644 index 00000000000..e70850db611 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/MonikerUtils.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// Utility class to detect and parse monikers from CODEOWNERS lines. + /// + public static class MonikerUtils + { + /// + /// Given a CODEOWNERS line, parse the moniker from the line if one exists. + /// + /// The CODEOWNERS line to parse. + /// String, the moniker if there was one on the line, null otherwise. + public static string ParseMonikerFromLine(string line) + { + if (line.StartsWith(SeparatorConstants.Comment)) + { + // Strip off the starting # and trim the result. Note, replacing tabs with + // spaces isn't necessary as Trim would trim off any leading or trailing tabs. + string strippedLine = line.Substring(1).Trim(); + var monikers = typeof(MonikerConstants) + .GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(field => field.IsLiteral) + .Where(field => field.FieldType == typeof(string)) + .Select(field => field.GetValue(null) as string); + foreach (string tempMoniker in monikers) + { + // Line starts with ":", unfortunately // has no colon and needs + // to be checked separately + if (strippedLine.StartsWith($"{tempMoniker}{SeparatorConstants.Colon}")) + { + return tempMoniker; + } + } + // Special case for the // moniker which has no colon + if (strippedLine.StartsWith($"{MonikerConstants.MissingFolder}")) + { + return MonikerConstants.MissingFolder; + } + } + // Anything that doesn't match an existing moniker is treated as a comment + return null; + } + + /// + /// Check whether a line is one of our Monikers. + /// + /// string, the line to check + /// true if the line contains a moniker, false otherwise + public static bool IsMonikerLine(string line) + { + if (line.StartsWith(SeparatorConstants.Comment)) + { + // Strip off the # + string strippedLine = line.Substring(1).Replace('\t', ' ').Trim(); + var monikers = typeof(MonikerConstants) + .GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(field => field.IsLiteral) + .Where(field => field.FieldType == typeof(string)) + .Select(field => field.GetValue(null) as string); + foreach (string tempMoniker in monikers) + { + // Line starts with ":", unfortunately // has no colon and needs + // to be checked separately + if (strippedLine.StartsWith($"{tempMoniker}{SeparatorConstants.Colon}")) + { + return true; + } + } + // Special case for the // moniker which has no colon + if (strippedLine.StartsWith($"{MonikerConstants.MissingFolder}")) + { + return true; + } + } + return false; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/OwnerDataUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/OwnerDataUtils.cs new file mode 100644 index 00000000000..3369022c951 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/OwnerDataUtils.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Caches; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// The OwnerData contains the team/user and user org visibility caches as well as methods used for owner (user or team) verification. + /// + public class OwnerDataUtils + { + private TeamUserCache _teamUserCache = null; + private UserOrgVisibilityCache _userOrgVisibilityCache = null; + + public OwnerDataUtils() + { + } + + /// + /// OwnerDataUtils constructor that takes a team/user blob storage URI override. This particular + /// ctor is used by parsing which doesn't do anything with user/org visibility data but is required + /// to expand teams. + /// + /// + public OwnerDataUtils(string teamUserBlobStorageUri) + { + _teamUserCache = new TeamUserCache(teamUserBlobStorageUri); + _userOrgVisibilityCache = new UserOrgVisibilityCache(null); + } + + /// + /// OwnerDataUtils that takes overrides for team/user and org/visibility data URI overrides. This + /// is used by linting which requires both to verify. + /// + /// + /// + public OwnerDataUtils(string teamUserBlobStorageUri, + string userOrgVisibilityBlobStorageUri) + { + _teamUserCache = new TeamUserCache(teamUserBlobStorageUri); + _userOrgVisibilityCache = new UserOrgVisibilityCache(userOrgVisibilityBlobStorageUri); + } + + public OwnerDataUtils(TeamUserCache teamUserCache, + UserOrgVisibilityCache userOrgVisibilityCache) + { + _teamUserCache = teamUserCache; + _userOrgVisibilityCache = userOrgVisibilityCache; + } + + /// + /// Check whether or not the owner has write permissions. + /// + /// The login of the owner to check + /// True if the owner has write permissions, false otherwise. + public bool IsWriteOwner(string owner) + { + return _userOrgVisibilityCache.UserOrgVisibilityDict.ContainsKey(owner); + } + + /// + /// Check whether or not the team has write permissions. + /// + /// The name of the team to check + /// True if the team has write permissions, false otherwise. + public bool IsWriteTeam(string team) + { + var teamWithoutOrg = team.Trim(); + if (teamWithoutOrg.Contains(SeparatorConstants.Team)) + { + teamWithoutOrg = teamWithoutOrg.Split(SeparatorConstants.Team, StringSplitOptions.TrimEntries)[1]; + } + return _teamUserCache.TeamUserDict.ContainsKey(teamWithoutOrg); + } + + /// + /// Check whether or not the user login is a member of of the Azure org + /// + /// The user login to check. + /// True, if the user is a member of the Azure org. False, if the user is not a member or not a public member + public bool IsPublicAzureMember(string login) + { + // If the user isn't in the dictionary then call to get it + if (_userOrgVisibilityCache.UserOrgVisibilityDict.ContainsKey(login)) + { + return _userOrgVisibilityCache.UserOrgVisibilityDict[login]; + } + return false; + } + + /// + /// If the team exists in the team user dictionary, return the list of users. + /// + /// The name of the team to expand + /// List>string< containing the users, empty list otherwise + public List ExpandTeam(string team) + { + var teamWithoutOrg = team.Trim(); + if (teamWithoutOrg.Contains(SeparatorConstants.Team)) + { + teamWithoutOrg = teamWithoutOrg.Split(SeparatorConstants.Team, StringSplitOptions.TrimEntries)[1]; + } + if (IsWriteTeam(teamWithoutOrg)) + { + return _teamUserCache.TeamUserDict[teamWithoutOrg]; + } + else + { + return new List(); + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/ParsingUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/ParsingUtils.cs new file mode 100644 index 00000000000..9cd330ea568 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/ParsingUtils.cs @@ -0,0 +1,231 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// The parsing utils contain the methods for parsing labels and owners from CODEOWNERS lines. It also + /// contains the method for determining whether or not a given line is a moniker or source path line (aka + /// something that needs to be parsed) + /// + public static class ParsingUtils + { + + /// + /// Parse the source path a CODEOWNERS line + /// + /// The line to parse labels from + /// The source path + public static string ParseSourcePathFromLine(string line) + { + string pathWithoutOwners = line; + // Owners, or lack thereof, are tracked elsewhere. + if (pathWithoutOwners.Contains(SeparatorConstants.Owner)) + { + // Grab the string up to the character before the owner constant + pathWithoutOwners = pathWithoutOwners.Substring(0, pathWithoutOwners.IndexOf(SeparatorConstants.Owner)); + } + pathWithoutOwners = pathWithoutOwners.Substring(0).Replace('\t', ' ').Trim(); + return pathWithoutOwners; + } + + /// + /// Parse the labels from a given CODEOWNERS line + /// + /// The line to parse labels from + /// Empty List<string> if there were no labels otherwise the List<string> containing the labels + public static List ParseLabelsFromLine(string line) + { + // This might look a bit odd but syntax for labels required they start with % because entries can contain + // multiple labels. If there's no % sign it's assumed that everything after the : is the label. + List labels = new List(); + string lineWithoutMoniker = line.Substring(line.IndexOf(SeparatorConstants.Colon) + 1).Trim(); + if (!string.IsNullOrWhiteSpace(lineWithoutMoniker)) + { + if (lineWithoutMoniker.Contains(SeparatorConstants.Label)) + { + labels.AddRange(lineWithoutMoniker.Split(SeparatorConstants.Label, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList()); + } + else + { + labels.Add(lineWithoutMoniker); + } + } + return labels; + } + + /// + /// Parse the owners from a given CODEOWNERS line + /// + /// The line to parse owners from + /// Whether or not to expand teams into their lists of owners when parsing. Linting will not expand teams but parsing will. + /// Empty List<string> if there were no users otherwise the List<string> containing the users + public static List ParseOwnersFromLine(OwnerDataUtils ownerData, string line, bool expandTeams) + { + // If there are no SeparatorConstants.Owner in the string then the string isn't formatted correctly. + // Every team/user needs to start with @ in the codeowners file. Split the codeownersLine on the + // SeparatorConstants.Owner character, the first entry is everything prior to the character and the users + // are all entries after that. + if (!line.Contains(SeparatorConstants.Owner)) + { + // return an empty the list + return new List(); + } + string justOwners = line.Substring(line.IndexOf(SeparatorConstants.Owner)); + List ownersWithoutTeamExpansion = justOwners.Split(SeparatorConstants.Owner, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + if (!expandTeams) + { + return ownersWithoutTeamExpansion; + } + List ownersWithTeamExpansion = new List(); + foreach (string owner in ownersWithoutTeamExpansion) + { + if (IsAzureTeam(owner)) + { + // If the list comes back empty it means it wasn't an azure-sdk-write team. At + // this point just add the non-expanded team entry which is the behavior of + // the parser today + var expandedTeam = ownerData.ExpandTeam(owner); + if (expandedTeam.Count == 0) + { + ownersWithTeamExpansion.Add(owner); + } + else + { + // Don't bother doing the union here to make the distinct list, + // that'll be done below + ownersWithTeamExpansion.AddRange(expandedTeam); + } + } + } + // Ensure that any owners that are on the line, which are also part of any teams on the line, only + // exist once in the list. Because git is case insensitive but case preserving the distinct list + // need to do the case insensitive comparison + return ownersWithTeamExpansion.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + } + + /// + /// Check whether or not a given CODEOWNERS line is a moniker or source line. + /// + /// string, the line to check + /// true if the line is a moniker or source line, false otherwise + public static bool IsMonikerOrSourceLine(string line) + { + // If the line is blank or whitespace. Note, + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + // if the line isn't blank or whitespace and isn't a comment then + // it's a source path line + else if (!line.StartsWith(SeparatorConstants.Comment)) + { + return true; + } + // At this point it's either a moniker or a comment + else + { + return MonikerUtils.IsMonikerLine(line); + } + } + + /// + /// Given a line, check to see if the line is a source path/owner line. Basically, any line in CODEOWNERS + /// that isn't a comment and isn't blank is considered to be a source path/owner line. + /// + /// string, the CODEOWNERS line to check + /// true, if the line is a source path/owner line, false otherwise + public static bool IsSourcePathOwnerLine(string line) + { + return !string.IsNullOrWhiteSpace(line) && + !line.StartsWith(SeparatorConstants.Comment); + } + + /// + /// Given the first line number of a block, find the block's end line number. The end block line number can be + /// the same as the start block line number if it's a source path line or a single, dangling moniker if the + /// block is malformatted. + /// + /// The starting line number of the block + /// The List<string> that represents the CODEOWNERS file + /// int, the line number of the block's end + public static int FindBlockEnd(int startBlockLineNumber, List codeownersFile) + { + // The block end will be a source path/owners line, the end of the file or the line prior to a blank line. + int endBlockLineNumber; + + for (endBlockLineNumber = startBlockLineNumber; endBlockLineNumber < codeownersFile.Count; endBlockLineNumber++) + { + string line = codeownersFile[endBlockLineNumber].Trim(); + // Blank lines aren't part of any block. If a blank line is encountered, the line prior to the blank line is + // the end of the block. + if (string.IsNullOrWhiteSpace(line)) + { + // This function needs to be called with the start of a block which is not a blank line. If things get + // here, the calling method is in error. + if (startBlockLineNumber == endBlockLineNumber) + { + throw new ArgumentException($"The block starting at line number, {startBlockLineNumber + 1}, was a blank line, which cannot the start of a block."); + } + endBlockLineNumber--; + break; + } + // If the line starts with a comment then it might not the end of the block, go to the next line + else if (line.StartsWith(SeparatorConstants.Comment)) + { + continue; + } + // If the line isn't null or whitespace and isn't a comment, then it's a source path/owner line + // which is the end of the block + else + { + break; + } + } + // This is the case where the very last line was a block or a source path/owner line. + // Set the end of the block to the last line. + if (endBlockLineNumber >= codeownersFile.Count) + { + endBlockLineNumber = codeownersFile.Count - 1; + } + return endBlockLineNumber; + } + + /// + /// Determine whether or not the owner is a team. Note, only Azure teams are allowed since all + /// teams in CODEOWNERS files need to be under azure-sdk-write. If the team name doesn't begin + /// with "Azure/" then it's not considered a team and later processing won't attempt to expand it. + /// + /// The owner to check + /// True if the owner is a GitHub Azure team, false otherwise. + public static bool IsAzureTeam(string owner) + { + + if (owner.StartsWith($"{OrgConstants.Azure}/{SeparatorConstants.Team}")) + { + return false; + } + return true; + } + + /// + /// Helper method to check if an owner is a GitHub team. This differs from the + /// OwnerIsAzureTeam, because it's only used to exclude owner aliases from + /// Codeowners entries. + /// + /// The owner to check. + /// True if it is a GitHub team, false otherwise. + public static bool IsGitHubTeam(string owner) + { + if (owner.Contains(SeparatorConstants.Team)) + { + return true; + } + return false; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/RepoLabelDataUtils.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/RepoLabelDataUtils.cs new file mode 100644 index 00000000000..cf54679c510 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/RepoLabelDataUtils.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils; +using Azure.Sdk.Tools.CodeownersUtils.Caches; + +namespace Azure.Sdk.Tools.CodeownersUtils.Utils +{ + /// + /// The RepoLabelData contains the repository label cache as well as methods used for repository label verification. + /// + public class RepoLabelDataUtils + { + private string _repository = null; + private RepoLabelCache _repoLabelCache = null; + public RepoLabelDataUtils() + { + } + + public RepoLabelDataUtils(string repoLabelBlobStorageUri, + string repository) + { + _repository = repository; + _repoLabelCache = new RepoLabelCache(repoLabelBlobStorageUri); + } + + // This constructor is for testing purposes only. + public RepoLabelDataUtils(RepoLabelCache repoLabelCache, + string repository) + { + _repository = repository; + _repoLabelCache = repoLabelCache; + } + + public bool LabelInRepo(string label) + { + return _repoLabelCache.RepoLabelDict[_repository].Contains(label); + } + + /// + /// Check to verify that repository label data exists. If it doesn't, that means this + /// is running in a repostiory it shouldn't be. + /// + /// True if label data exists for the repository. + public bool RepoLabelDataExists() + { + return _repoLabelCache.RepoLabelDict.ContainsKey(_repository); + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/CodeownersLinter.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/CodeownersLinter.cs new file mode 100644 index 00000000000..55d1acb75c2 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/CodeownersLinter.cs @@ -0,0 +1,325 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Verification +{ + /// + /// The primary entry point for Linting a CODEOWNERS file + /// + public static class CodeownersLinter + { + /// + /// Load the Codeowners file and process it a block at a time + /// + /// Directory utils used for source path entry verification. + /// Owner data used for owner verification. + /// Repository label data used for label verification. + /// Codeowners file with full path + /// + public static List LintCodeownersFile(DirectoryUtils directoryUtils, + OwnerDataUtils ownerData, + RepoLabelDataUtils repoLabelData, + string codeownersFileFullPath) + { + List errors = new List(); + // Load the codeowners file and process it a block at a time + List codeownersFile = FileHelpers.LoadFileAsStringList(codeownersFileFullPath); + + // Start parsing the codeowners file, a block at a time. + // A block can be one of the following: + // 1. A single source path/owner line + // 2. One or more monikers that either ends in source path/owner line or a blank line, depending + // on the moniker. + for (int currentLineNum = 0; currentLineNum < codeownersFile.Count; currentLineNum++) + { + string line = codeownersFile[currentLineNum]; + if (ParsingUtils.IsMonikerOrSourceLine(line)) + { + // A block can be a single line, if it's a source path/owners line, or if the block starts + // with a moniker, it'll be multi-line + int blockEnd = ParsingUtils.FindBlockEnd(currentLineNum, codeownersFile); + VerifyBlock(directoryUtils, + ownerData, + repoLabelData, + errors, + currentLineNum, + blockEnd, + codeownersFile); + // After processing the block, set the current line to the end line which will get + // incremented and continue the processing the line after the block + currentLineNum = blockEnd; + } + } + return errors; + } + + /// + /// Basically a call to VerifyBlock that'll skip single line verification. This is used by the parser + /// which only needs to know if there are block errors. Because there's no single line verification + /// the DirectoryUtils, OwnerDataUtils and RepoLabelDataUtils are unnecessary and can be null. + /// + /// List of errors that will be appended to if any are found with the block + /// The line number which is the start of the block to verify + /// Int, the line number that marks the of the block + /// The List<string> that represents the CODEOWNERS file + public static void VerifyBlock(List errors, + int startBlockLineNumber, + int endBlockLineNumber, + List codeownersFile) + { + VerifyBlock(null, + null, + null, + errors, + startBlockLineNumber, + endBlockLineNumber, + codeownersFile, + false /* don't do single line verification */ ); + } + + /// + /// Verify the formatting of a block in codeowners. + /// Definitions: + /// Source path/Owner Line: Any line in CODEOWNERS that is not a comment and not blank. + /// Metadata block : A metadata block is a block that consists of one or more metadata tags which, depending on the tags, + /// may end with a source path/owner line. + /// + /// Owner data used for owner verification. + /// Owner data used for owner verification. + /// Repository label data used for label verification. + /// List of errors that will be appended to if any are found with the block + /// The line number which is the start of the block to verify + /// Int, the line number that marks the of the block + /// The List<string> that represents the CODEOWNERS file + /// Whether or not to perform single line verification, default is true. The + /// reason this would be turned off would be parsing, which just needs to verify the block is good. + public static void VerifyBlock(DirectoryUtils directoryUtils, + OwnerDataUtils ownerData, + RepoLabelDataUtils repoLabelData, + List errors, + int startBlockLineNumber, + int endBlockLineNumber, + List codeownersFile, + bool singleLineVerification = true) + { + List blockErrorStrings = new List(); + // The codeownersFile as a list is 0 based, for reporting purposes it needs + // to be 1 based to match the exact line in the CODEOWNERS file. + int startLineNumberForReporting = startBlockLineNumber + 1; + int endLineNumberForReporting = endBlockLineNumber + 1; + bool endsWithSourceOwnerLine = ParsingUtils.IsSourcePathOwnerLine(codeownersFile[endBlockLineNumber]); + // Booleans for every moniker, will be set to true when found, are used to verify the block + // contains what it needs to contain for the monikers found within it. + bool blockHasAzureSdkOwners = false; + bool blockHasMissingFolder = false; + bool blockHasPRLabel = false; + bool blockHasServiceLabel = false; + bool blockHasServiceOwners = false; + + for (int blockLine = startBlockLineNumber; blockLine <= endBlockLineNumber; blockLine++) + { + string line = codeownersFile[blockLine]; + int lineNumberForReporting = blockLine + 1; + bool isSourcePathOwnerLine = ParsingUtils.IsSourcePathOwnerLine(line); + if (isSourcePathOwnerLine) + { + if (singleLineVerification) + { + VerifySingleLine(directoryUtils, + ownerData, + repoLabelData, + errors, + lineNumberForReporting, + line, + isSourcePathOwnerLine, + !endsWithSourceOwnerLine); + } + } + else + { + string moniker = MonikerUtils.ParseMonikerFromLine(line); + // This can happen if there's a comment line in the block, skip the line + if (null == moniker) + { + continue; + } + switch (moniker) + { + case MonikerConstants.AzureSdkOwners: + if (blockHasAzureSdkOwners) + { + blockErrorStrings.Add($"{MonikerConstants.AzureSdkOwners}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}"); + } + blockHasAzureSdkOwners = true; + break; + case MonikerConstants.PRLabel: + if (blockHasPRLabel) + { + blockErrorStrings.Add($"{MonikerConstants.PRLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}"); + } + blockHasPRLabel = true; + break; + case MonikerConstants.MissingFolder: + if (blockHasMissingFolder) + { + blockErrorStrings.Add($"{MonikerConstants.MissingFolder}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}"); + } + blockHasMissingFolder = true; + break; + case MonikerConstants.ServiceLabel: + if (blockHasServiceLabel) + { + blockErrorStrings.Add($"{MonikerConstants.ServiceLabel}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}"); + } + blockHasServiceLabel = true; + break; + case MonikerConstants.ServiceOwners: + if (blockHasServiceOwners) + { + blockErrorStrings.Add($"{MonikerConstants.ServiceOwners}{ErrorMessageConstants.DuplicateMonikerInBlockPartial}"); + } + blockHasServiceOwners = true; + break; + default: + // This shouldn't get here unless someone adds a new moniker and forgets to add it to the switch statement + throw new ArgumentException($"Unexpected moniker '{moniker}' found on line {lineNumberForReporting}\nLine={line}"); + } + + if (singleLineVerification) + { + VerifySingleLine(directoryUtils, + ownerData, + repoLabelData, + errors, + lineNumberForReporting, + line, + isSourcePathOwnerLine, + !endsWithSourceOwnerLine, // If the block ends in a source path/owner line then we don't expect owners on moniker lines + moniker); + } + } + } + + // After the block has been processed, ensure that any monikers are paired correctly with other + // monikers or source path/owners + + // If the block is a single source path/owners line then there's nothing else to be done since there + // can't be any block errors. + if (startBlockLineNumber == endBlockLineNumber && endsWithSourceOwnerLine) + { + return; + } + + // AzureSdkOwners must be part of a block of that a ServiceLabel entry as the AzureSdkOwners are associated with + // that ServiceLabel + if (blockHasAzureSdkOwners && !blockHasServiceLabel) + { + blockErrorStrings.Add(ErrorMessageConstants.AzureSdkOwnersMustBeWithServiceLabel); + } + + if (blockHasServiceOwners && !blockHasServiceLabel) + { + blockErrorStrings.Add(ErrorMessageConstants.ServiceOwnersMustBeWithServiceLabel); + } + + // PRLabel moniker must be in a block that ends with a source path/owner line + if (blockHasPRLabel && !endsWithSourceOwnerLine) + { + blockErrorStrings.Add($"{MonikerConstants.PRLabel}{ErrorMessageConstants.NeedsToEndWithSourceOwnerPartial}"); + } + + // ServiceLabel needs to be part of a block that has one of, ServiceOwners or #// (MonikerConstants.MissingFolder), + // or ends in a source path/owner line both not both. + if (blockHasServiceLabel) + { + if (!endsWithSourceOwnerLine && !blockHasServiceOwners && !blockHasMissingFolder) + { + blockErrorStrings.Add(ErrorMessageConstants.ServiceLabelNeedsOwners); + } + else if (endsWithSourceOwnerLine && (blockHasServiceOwners || blockHasMissingFolder)) + { + blockErrorStrings.Add(ErrorMessageConstants.ServiceLabelHasTooManyOwners); + } + else if (blockHasServiceOwners && blockHasMissingFolder) + { + blockErrorStrings.Add(ErrorMessageConstants.ServiceLabelHasTooManyOwnerMonikers); + } + } + + if (blockErrorStrings.Count > 0) + { + List blockLines = new List(); + blockLines.AddRange(codeownersFile.GetRange(startBlockLineNumber, (endBlockLineNumber - startBlockLineNumber) + 1)); + errors.Add(new BlockFormattingError(startLineNumberForReporting, + endLineNumberForReporting, + blockLines, + blockErrorStrings)); + + } + } + + /// + /// Verify the contents of a single line, called as part of the block processing. + /// + /// Owner data used for owner verification. + /// Repository label data used for label verification. + /// List of errors that will be appended to if any are found with the block + /// The line number, for reporting purposes, of the line being processed. + /// The CODEOWNERS line to process. + /// True if owners are expected on a moniker line. This would be true if the moniker is part of a block that didn't end in a source path/owner line. + public static void VerifySingleLine(DirectoryUtils directoryUtils, + OwnerDataUtils ownerData, + RepoLabelDataUtils repoLabelData, + List errors, + int lineNumberForReporting, + string line, + bool isSourcePathOwnerLine, + bool expectOwnersIfMoniker, + string moniker = null) + { + List errorStrings = new List(); + if (isSourcePathOwnerLine) + { + // Verify the source path and owners + directoryUtils.VerifySourcePathEntry(line, errorStrings); + Owners.VerifyOwners(ownerData, line, true, errorStrings); + } + else + { + // At this point, the moniker shouldn't be null, comment lines should have been + // sifted out by the calling method. + if (null != moniker) + { + switch (moniker) + { + // Both ServiceLabel and blockHasPRLabel moniker lines need to have labels + case MonikerConstants.ServiceLabel: + case MonikerConstants.PRLabel: + Labels.VerifyLabels(repoLabelData, line, moniker, errorStrings); + break; + // MissingFolder, ServiceOwners and AzureSdkOwners + case MonikerConstants.MissingFolder: + case MonikerConstants.ServiceOwners: + case MonikerConstants.AzureSdkOwners: + Owners.VerifyOwners(ownerData, line, expectOwnersIfMoniker, errorStrings); + break; + default: + // This shouldn't get here unless someone adds a new moniker and forgets to add it to the switch statement + throw new ArgumentException($"Unexpected moniker '{moniker}' found on line {lineNumberForReporting}\nLine={line}"); + } + } + } + // If any errors were encountered on the line, create a new exception and add it to the list + if (errorStrings.Count > 0) + { + errors.Add(new SingleLineError(lineNumberForReporting, line, errorStrings)); + } + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Labels.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Labels.cs new file mode 100644 index 00000000000..26e93ba3cbc --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Labels.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Verification +{ + /// + /// Verification class for CODEOWNERS lines that contain labels + /// + public static class Labels + { + /// + /// Verify that the labels, on a given CODEOWNERS metadata line, are defined for the repository. + /// + /// The repository label data + /// The CODEOWNERS line to parse + /// The moniker being processed. Necessary for determining the number of allowed labels. + /// List <string>, any error strings are added to this list. + public static void VerifyLabels(RepoLabelDataUtils repoLabelData, string line, string moniker, List errorStrings) + { + List labels = ParsingUtils.ParseLabelsFromLine(line); + + if (labels.Count == 0) + { + errorStrings.Add(ErrorMessageConstants.MissingLabelForMoniker); + } + + // The Service Attention label is not valid in a PRLabel moniker + if (MonikerConstants.PRLabel == moniker) + { + // Regardless of the number of labels on the moniker, ServiceAttention should not be there. + if (labels.Contains(LabelConstants.ServiceAttention, StringComparer.InvariantCultureIgnoreCase)) + { + errorStrings.Add(ErrorMessageConstants.ServiceAttentionIsNotAValidPRLabel); + } + } + // ServiceLabel cannot only have Service Attention + else if (MonikerConstants.ServiceLabel == moniker) + { + if (labels.Count == 1 && labels.Contains(LabelConstants.ServiceAttention, StringComparer.InvariantCultureIgnoreCase)) + { + errorStrings.Add(ErrorMessageConstants.ServiceLabelMustContainAServiceLabel); + } + } + + // Verify each label in the list exists + foreach (string label in labels) + { + if (!repoLabelData.LabelInRepo(label)) + { + errorStrings.Add($"'{label}'{ErrorMessageConstants.InvalidRepositoryLabelPartial}"); + } + } + return; + } + } +} diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Owners.cs b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Owners.cs new file mode 100644 index 00000000000..fd339477364 --- /dev/null +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Verification/Owners.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeownersUtils.Constants; +using Azure.Sdk.Tools.CodeownersUtils.Errors; +using Azure.Sdk.Tools.CodeownersUtils.Utils; + +namespace Azure.Sdk.Tools.CodeownersUtils.Verification +{ + /// + /// Verification class for CODEOWNERS lines containing owners. + /// + public static class Owners + { + /// + /// Verify the owners (teams or users) for a given CODEOWNERS line + /// + /// OwnerData instance + /// The CODEOWNERS line being parsed + /// Whether or not owners are expected. Some monikers may or may not have owners if their block ends in a source path/owner line. + /// List of errors belonging to the current line. New errors are added to the list. + public static void VerifyOwners(OwnerDataUtils ownerData, string line, bool expectOwners, List errorStrings) + { + List ownerList = ParsingUtils.ParseOwnersFromLine(ownerData, line, false /* teams aren't expanded for linting */); + // Some CODEOWNERS lines require owners to be on the line, like source path/owners lines. Some CODEOWNERS + // monikers, eg. AzureSdkOwners, may or may not have owners depending on whether or not the block they're + // part of ends in a source path/owners line. If the line doesn't contain the Owner Separator and owners + // are expected, then add the error and return, otherwise just return. + if (ownerList.Count == 0) + { + if (expectOwners) + { + errorStrings.Add(ErrorMessageConstants.NoOwnersDefined); + } + return; + } + + // Verify that each owner exists and has write permissions + foreach (string owner in ownerList) + { + // If the owner is a team then it needs to have write permission and needs to be + // one of the teams in the teamUser data. This is the same for metadata lines or + // path lines in the CODEOWNERS file + if (owner.StartsWith(OrgConstants.AzureOrgTeamConstant, StringComparison.OrdinalIgnoreCase)) + { + // Ensure the team has write permission + if (!ownerData.IsWriteTeam(owner)) + { + errorStrings.Add($"{owner}{ErrorMessageConstants.InvalidTeamPartial}"); + } + } + // else, the owner is a user or a malformed team entry + else + { + // The list of sourcepath line owners comes directly from the CODEOWNERS errors which means the only owners being processed + // are ones that have already been flagged as errors. This means that only the following checks need to be done. + // 1. If an owner has write permission and but isn't a member of Azure, it means the owner's membership in Azure isn't public + // and needs to be. + // 2. If the owner doesn't have write permission: + // a. Check whether or not the owner is a malformed team entry (an entry that doesn't start with @Azure/) + // If the owner has write permission + if (ownerData.IsWriteOwner(owner)) + { + // Verify that the owner is a public member of Azure. Github's parsing of CODEOWNERS doesn't recognize + // an owner unless their azure membership is public + if (!ownerData.IsPublicAzureMember(owner)) + { + errorStrings.Add($"{owner}{ErrorMessageConstants.NotAPublicMemberOfAzurePartial}"); + } + } + else + { + // if the owner is not a write user, check and see if the entry is a malformed team entry, missing the "@Azure/" + if (ownerData.IsWriteTeam(owner)) + { + errorStrings.Add($"{owner}{ErrorMessageConstants.MalformedTeamEntryPartial}"); + } + else + { + // At this point whomever they are they're not: + // 1. a team under azure-sdk-write + // 2. a malformed team entry, for a team under azure-sdk-write + // 3. a user with write permissions (would be part of the distinct list of all users from all + // teams under azure-sdk-write) + // It's unclear, with the data we have, if they're even a valid GitHub user and, if so, if + // they're even part of Azure. + errorStrings.Add($"{owner}{ErrorMessageConstants.InvalidUserPartial}"); + } + } + } + } + } + } +} diff --git a/tools/codeowners-utils/CodeownersUtils.sln b/tools/codeowners-utils/CodeownersUtils.sln new file mode 100644 index 00000000000..2c9944779d5 --- /dev/null +++ b/tools/codeowners-utils/CodeownersUtils.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.6.33815.320 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeownersLinter", "Azure.Sdk.Tools.CodeownersLinter\Azure.Sdk.Tools.CodeownersLinter.csproj", "{A24C1588-EEC9-4421-8E0F-1C167B0F2112}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeownersUtils.Tests", "Azure.Sdk.Tools.CodeownersLinter.Tests\Azure.Sdk.Tools.CodeownersUtils.Tests.csproj", "{50856BB3-8AF2-455F-ABC9-5EF1297C2917}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Sdk.Tools.CodeownersUtils", "Azure.Sdk.Tools.CodeownersUtils\Azure.Sdk.Tools.CodeownersUtils.csproj", "{0F5654AF-639B-4005-B572-81CBDCB5B845}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A24C1588-EEC9-4421-8E0F-1C167B0F2112}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A24C1588-EEC9-4421-8E0F-1C167B0F2112}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A24C1588-EEC9-4421-8E0F-1C167B0F2112}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A24C1588-EEC9-4421-8E0F-1C167B0F2112}.Release|Any CPU.Build.0 = Release|Any CPU + {50856BB3-8AF2-455F-ABC9-5EF1297C2917}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50856BB3-8AF2-455F-ABC9-5EF1297C2917}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50856BB3-8AF2-455F-ABC9-5EF1297C2917}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50856BB3-8AF2-455F-ABC9-5EF1297C2917}.Release|Any CPU.Build.0 = Release|Any CPU + {0F5654AF-639B-4005-B572-81CBDCB5B845}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0F5654AF-639B-4005-B572-81CBDCB5B845}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0F5654AF-639B-4005-B572-81CBDCB5B845}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0F5654AF-639B-4005-B572-81CBDCB5B845}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {50F82DFE-95E9-4C82-8370-824A48AACA61} + EndGlobalSection +EndGlobal diff --git a/tools/codeowners-utils/METADATA.md b/tools/codeowners-utils/METADATA.md new file mode 100644 index 00000000000..3d34a502c10 --- /dev/null +++ b/tools/codeowners-utils/METADATA.md @@ -0,0 +1,93 @@ +# CODEOWNERS Metadata + +Azure-sdk* repository CODEOWNERS files contain data beyond the normal source path/owner lines. This metadata is used by workflows for processing. The most notable consumer is [github-event-processor](https://github.com/Azure/azure-sdk-tools/tree/main/tools/github-event-processor) which uses these when processing certain GitHub Actions. The full set of action processing rules is documented [here](https://github.com/Azure/azure-sdk-tools/blob/main/tools/github-event-processor/RULES.md). The common definitions used in this document are as follows: + +- **Moniker** - Any of the existing tags we currently support. PRLabel or ServiceLabel would be examples of this. +- **Metadata block** - A block containing one or more metadata tags and/or a source path/owner line. A block ends with a blank line or _a single source path/owner line_. This means that one more monikers followed by multiple source path/owner lines will only apply the the metadata to the first source path/owner line. Each and every one source path/owner line requires its own metadata monikers. +- **Source path/owner line** - A single source path/owner line in CODEOWNERS is its own block. There are some metadata tags that must be part of a block that ends in source path/owner line and some tags that can be. These are defined below. + +## Metadata Monikers + +- **AzureSdkOwners:** - This moniker is used to denote Azure Sdk owners that have triage responsibility for a given service label. This moniker must be part of a block containing a ServiceLabel entry. +- **PRLabel:** - This moniker is used by workflows to determine what label(s) will get added to a pull request based upon the file paths of the files in the pull request. This moniker must be part of a block that ends in a source path/owner line. +- **ServiceLabel:** - This moniker contains the service label that's used to figure out what users need to be @ mentioned in an issue when the Service Attention label is added. This moniker must be part of a block that either ends in a source path/owner line, the ServiceOwners moniker or the /<NotInRepo>/ moniker. If the ServiceLabel is part of a block that ends in the source path/owner line, the service owners are inferred from that. +- **ServiceOwners:** - The moniker is used to identify the owners associated with a service label if the service label isn't part of a block ending in a source path/owner line. This moniker cannot be part of a source path/owner line. +- **/<NotInRepo>/** - This is the existing moniker used to denote service owners in CODEOWNERS files. This will ultimately be replaced by the ServiceOwners moniker, which more clearly defines what it actually is, but right now the parser and linter will handle both. This moniker cannot be part of a source path/owner line. Also, this is the only moniker that doesn't have a colon separator after the moniker, before the labels. + +## Examples of moniker usage in CODEOWNERS files + +This list of examples is exhaustive. If an example isn't in here then it won't work and the linter will catch it. + +- A single source path/owner line is its own block. + +```text + +# Optional comment +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 + +``` + +- `AzureSdkOwners` must be part of a block that contains a ServiceLabel entry. If that block ends in a source path/owner line, and the AzureSdkOwners entry is empty, it'll have the same owners that are only the source path/owner line. If it's part of block that contains a ServiceLabel/ServiceOwner combination, then it must have it's own owners defined. + +```text + +# AzureSdkOwners: @fakeUser3 @fakeUser4 +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 +OR +# AzureSdkOwners: +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 +OR +# AzureSdkOwners: @fakeUser3 @fakeUser4 +# ServiceLabel: %fakeLabel12 +# ServiceOwners: @fakeUser1 @fakeUser2 + +``` + +- `PRLabel` must be part of a block that ends in a source path/owner + +```text + +# PRLabel: %Label1 %Label2 +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 + +``` + +- If a `ServiceLabel` is part of a block that ends in a source path/owner line, the ServiceOwners will be the inferred to be the same as the source owners. A ServiceLabel ending in a source path/owners block cannot have a ServiceOwners entry because the entire reason it's part of that block is because the service owners and source owners are the same. _This is only time that a moniker not explicitly in the block will have inferred data._ + +```text + +# ServiceLabel: %Label1 %Label2 +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 + +``` + +- If a `ServiceLabel` is not part of a block that ends in a source path/owner line, then it must be part of a two line block consisting only of a ServiceLabel and either a ServiceOwners or /<NotInRepo>/. New entries should use ServiceOwners. + +```text + +# ServiceLabel: %Label1 %Label2 +# ServiceOwners: @fakeUser1 @Azure/fakeTeam1 +OR +# ServiceLabel: %Label1 %Label2 +# /<NotInRepo>/ @fakeUser1 @Azure/fakeTeam1 + +``` + +- This might look complex but there are really only 2 types of blocks. The first ends with a source path/owner line and may have AzureSdkOwners, ServiceLabel PRLabel entries and the second is ServiceLabel/ServiceOwner block which may have AzureSdkOwners. + +```text + +# AzureSdkOwners: (optional) +# ServiceLabel: (optional) +# PRLabel: (optional) +/sdk/SomePath/ @fakeUser1 @fakeUser2 @Azure/fakeTeam1 + +``` + +```text + +# AzureSdkOwners: (optional) +# ServiceLabel: %Label1 %Label2 +# ServiceOwners: @fakeUser1 @Azure/fakeTeam1 + +``` diff --git a/tools/codeowners-utils/README.md b/tools/codeowners-utils/README.md new file mode 100644 index 00000000000..0290bf16053 --- /dev/null +++ b/tools/codeowners-utils/README.md @@ -0,0 +1,111 @@ + +# Codeowners-Utils + +Codeowners Utils contains utilities to both parse and lint CODEOWNERS files in Azure Sdk repositories. The reason why this is necessary, instead just using GitHub's CODEOWNERS validation, is that we have our own type of metadata, which exists in comments in the CODEOWNERS files, which is something that requires additional validation. [Metadata - definition, usage and block structure can be found here](./METADATA.md) + +## Parsing + +The entry points for parsing and matching paths against CODEOWNERS entries exist in [CodeownersParser](./Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersParser.cs). The CODEOWNERS data is parsed into a list of[CodeownersEntry](./Azure.Sdk.Tools.CodeownersUtils/Parsing/CodeownersEntry.cs). ParseCodeownersFile requires the file or URL of the CODEOWNERS file to parse and takes an optional URI to override the default team/user data URI which is used to expand teams in CODEOWNERS entries. The resulting list entries are in the same order as they are in the CODEOWNERS file. + + ```csharp + List codeownersEntries = CodeownersParser.ParseCodeownersFile(codeownersUrl, [teamStorageURI]); + ``` + +The resulting list of CodeownersEntry objects are typically used to get the matching CodeownersEntry for a given repository file. There are several tools that use this information today like github-event-processor, notification-configuration, and pipeline-owners-extractor. Like GitHub, it matches things in reverse order, meaning that it starts from the end of the list and returns the first match that it finds or an empty entry if there was no match. + +```csharp +string buildDefPath = "tools/github-event-processor/ci.yml"; +CodeownersEntry codeownersEntry = CodeownersParser.GetMatchingCodeownersEntry(buildDefPath, codeownersEntries); +``` + +## CodeownersEntry object + +A CodeownersEntry object has the following members: + +- **PathExpression** - If the CODEOWNERS metadata block ended in a source path/owner line, this is the path portion. For example, `sdk/ServiceDirectory1` or `/sdk/**/azure-myservice-*/` etc. This is empty if the meta +- **SourceOwners** - The list of owners for a given source path/owner line. This is empty if there are no owners or, the metadata block did not end in a source path/owner line. +- **PRLabels** - The list of labels parsed from the `PRLabel` metadata moniker or empty if metadata block did not contain the moniker. +- **AzureSdkOwners** - The list of owners parsed from the `AzureSdkOwner` metadata moniker, empty if metadata block did not contain the moniker or, if the moniker had no users defined but was part of a block that ended in a source path/owners line, this list would contain the same owners as the `SourceOwners`. +- **ServiceLabels** - The list of labels parsed from the `ServiceLabel` metadata moniker or empty if metadata block did not contain the moniker. +- **ServiceOwners** - The list of owners parsed from the `ServiceOwner` metadata moniker. Empty if metadata block did not contain the moniker or, if the ServiceLabel is part of a block that ends in a source path/owner line, these owners will be the same as the `SourceOwners`. + +Additional note about owners and parsing: The team/user data is used to expand teams into a list of users but in order to do so, the team needs to be a child team of azure-sdk-write. The reasoning for this is twofold, the first being that the GitHub criteria for an owner in a CODEOWNERS file is that the owner must have write permission with the second being that all teams with write permissions for our repositories are children of azure-sdk-write. Similarly, in order to have write permission for Azure/azure-sdk* repositories, individual owners must be direct members of azure-sdk-write or one of its child teams. + +## Linting + +This tool will analyze an azure-sdk* CODEOWNERS file, including our specific brand of Metadata, to ensure correctness. It'll output errors in metadata blocks as well as errors for any individual lines within a metadata block. + +### What is verified during linting? + +- **Owners** + - Users and Teams are verified to have write permissions. + - Users are also verified to be Public members of Azure. This documented in the [azure-sdk onboarding docs for acess](https://eng.ms/docs/products/azure-developer-experience/onboard/access). This is necessary for tooling in order to be able to determine Azure org membership for a given user. This cannot be done if the user's Azure membership is private and the tooling will process them as if they weren't a member of Azure. + - Malformed team entries, entries missing the prepended org `@Azure/` can be detected but only if they're child teams of azure-sdk-write. +- **Labels** + - Whether or not the label exists in a particular repository. +- **Source paths** (not something GitHub verifies) + - Does the path (directory or file) exist? + - If the path is a glob + - Is the glob syntactically valid? Does it contain [forbidden characters](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax) or is it malformed? + - Does the glob have any matches in the repository? +- **Metadata block formatting** + - Verifies the correctness of metadata blocks according our [Metadata definitions](./METADATA.md). + +### Errors + +There are two types of errors, single line errors and block formatting errors. + +- **Single line error** - These verification errors for a single CODEOWNERS line. The error message will contain the line number, the source line and all of the errors. For example: + +```text +Error(s) on line 125 +Source Line: /sdk/SomeServiceDirectory/BadPath @fakeUser1 @FakeTeam @fakeUser2 + -/sdk/SomeServiceDirectory/BadPath path or file does not exist in repository. + -fakeUser1 is not a public member of Azure. + -FakeTeam is a malformed team entry and should start with '@Azure/'. + -fakeUser2 is an invalid user. Ensure the user exists, is public member of Azure and has write permissions. +``` + +- **Block formatting error** - These are errors with the formatting of the metadata block. The error message will clearly identify that it's a block error and will contain the start/end line numbers as well as the contents of the block. In the example below, the ServiceLabel entry would be associated with the source path/owners line if it weren't commented out. A ServiceLabel needs owners. + +```text +Source block error. +Source Block Start: 728 + # ServiceLabel: %FakeLabel1 %Service Attention + #/sdk/SomeServiceDirectory/ @fakeUser3 +Source Block End: 729 + -ServiceLabel needs to be followed by, // or ServiceOwners with owners, or a source path/owner line. +``` + +### Where will this run? + +Right now, plan is to have this running in same set of repositories that the github-event-processor runs in it's this set of repositories that utilize CODEOWNERS data along with our flavor of metadata. There will be a nightly pipeline run but the pipeline will also run on pull requests with CODEOWNERS changes. _Note: GitHub's linting might say a particular CODEOWNERS file is clean, that's not necessarily true with our metadata._ + +#### What about the existing errors? + +Every repository that this will be enabled for will have a baseline file, `CODEOWNERS_baseline_errors.txt`, sitting next to its CODEOWNERS file if there are existing errors. This file is a deduplicated list of known single line errors, without the line and line numbers, and can be used by the linter to filter its output. This will prevent new errors from getting into the file as well as allow the nightly pipeline runs to start out passing. The downside of this is that if someone adds a new line that contains an existing error it wouldn't get caught. The repository owners will ultimately be responsible for cleaning out these errors and when they're all cleaned up, the file is removed. To the linter, no baseline file means no existing errors. + +#### Why only filter Single errors without the line and line number? + +Line numbers become moot as soon as someone adds a line to the CODEOWNERS file and there's really no good way deal with this. Further, single line errors contain errors that pertain to an invalid path, invalid owners, malformed team names, invalid labels etc. and these are type of errors that can exist multiple times in a file. A path should only exist once in a CODEOWNERS file but owners and labels can exist multiple times. For example, if @owner1 isn't a public member of Azure and exists as an owner for multiple items all of those errors are filtered out with the 1 deduplicated line. Block formatting errors, on the other hand, are errors that are very specific to a given block and if parsing encounters a block with errors, that entry is thrown out. If block formatting errors were deduplicated, someone adding a new block could copy and paste from a block with formatting errors. Because errors are deduplicated, the pipeline running for the CODEOWNERS changes wouldn't report the block error and something down the line using the parser, wouldn't work as expected because the block was bad and there's no parsed entry for it. The exiting repositories, that the linter will be running in, have no CODEOWNERS block errors and this will prevent them from being added. + +### Where does the team/user and label data come from + +The team/user, org visibility and repository label data are stored in Azure Blob Storage. The data is populated by the [github-team-user-store](https://github.com/Azure/azure-sdk-tools/tree/main/tools/github-team-user-store) which runs daily as part of the pipeline-owners-extractor. _Note: To be very clear, this information is not secret nor is it exposing anything that can't already be gleaned from the existing CODEOWNERS files or, in the case of labels, inspecting the repository._ This has to be done this way for the following reasons: + +- Fetching this information requires a specific GitHub token with specific permissions which cannot be granted GitHub workflows. +- This is going to run as part of a nightly run as well as a CI pipeline when changes to CODEOWNERS files are made and public pipelines cannot have variables. + +This also means that certain changes, like a user changing their Azure membership to public or a new label was added to the repository or a new team was added, won't be immediately picked up. The pipeline-owners-extractor would need to run so the pre-populated data reflects them. + +### Codeowners Utils has following requirements and dependencies + +1. **Linter only** Must be run in a full repository, not a sparse checked out one. The reason for this is that paths in the CODEOWNERS are verified to exist and, if they're a glob path, they actually have matches in the repository. +2. Microsoft.Extensions.FileSystemGlobbing. This is used by the linter to verify the paths in CODEOWNERS exist or, if globs, they have matches. Used by parser's GetMatchingCodeownersEntry to determine whether or not a CodeownersEntry matches the target path of what's passed in. + +In addition to the above dependencies, the Azure.Sdk.Tools.CodeownersUtils.Tests project has the following additional test dependencies. + +1. NUnit - NUnit unit testing +2. NUnit3TestAdapter - Used to run NUnit3 tests in Visual Studio +3. Microsoft.NET.Test.Sdk - For the ability to run tests using **dotnet test** +4. coverlet.collector - Generates test coverage data diff --git a/tools/codeowners-utils/ci.yml b/tools/codeowners-utils/ci.yml new file mode 100644 index 00000000000..b4f2b3956e3 --- /dev/null +++ b/tools/codeowners-utils/ci.yml @@ -0,0 +1,24 @@ +# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. +trigger: + branches: + include: + - main + paths: + include: + - tools/codeowners-utils + +pr: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - tools/codeowners-utils + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml + parameters: + ToolDirectory: tools/codeowners-utils From e227149b367dc7f3dd4ebcc2f22e97b99f9f6b91 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 19 Oct 2023 19:00:15 -0400 Subject: [PATCH 75/93] Fix update test resources tagging when tags are empty. Bump max limit (#7171) --- eng/common/TestResources/Update-TestResources.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/TestResources/Update-TestResources.ps1 b/eng/common/TestResources/Update-TestResources.ps1 index a9983f547a9..9592e8a8caa 100644 --- a/eng/common/TestResources/Update-TestResources.ps1 +++ b/eng/common/TestResources/Update-TestResources.ps1 @@ -26,7 +26,7 @@ param ( [string] $SubscriptionId, [Parameter()] - [ValidateRange(1, 7*24)] + [ValidateRange(1, 30*24)] [int] $DeleteAfterHours = 48 ) @@ -136,6 +136,9 @@ try { Log "Updating DeleteAfter to '$deleteAfter'" Write-Warning "Any clean-up scripts running against subscription '$SubscriptionId' may delete resource group '$ResourceGroupName' after $DeleteAfterHours hours." + if (!$resourceGroup.Tags) { + $resourceGroup.Tags = @{} + } $resourceGroup.Tags['DeleteAfter'] = $deleteAfter Log "Updating resource group '$ResourceGroupName'" From 2336198a4cbd3eaa3a05b5ba3ae35d24d05cbf5e Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 20 Oct 2023 13:49:18 -0400 Subject: [PATCH 76/93] Add max lifespan parameter to resource groups (#7160) * Add max lifespan parameter to resource groups * Update docs * Only delete ARM deployments containing outputs * Delete arm deployments in main test sub * Fix up immutability policy clean up for blobs * Flexible deletion timing --- doc/engsys_resource_management.md | 3 ++ eng/pipelines/live-test-cleanup.yml | 5 ++- eng/scripts/Remove-WormStorageAccounts.ps1 | 4 +- eng/scripts/live-test-resource-cleanup.ps1 | 51 ++++++++++++++++------ 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/doc/engsys_resource_management.md b/doc/engsys_resource_management.md index 0bc706115f4..2d558d09ba5 100644 --- a/doc/engsys_resource_management.md +++ b/doc/engsys_resource_management.md @@ -43,6 +43,9 @@ If you need the group for a little more time, add/update a tag named `DeleteAfte *To create long-lived resources:* +NOTE: Resource groups in the playground subscription, if compliant and marked with valid aliases (see below), **will still +be marked for deletion after 30 days.** See the below examples for how to extend this deletion deadline if necessary. + Create a resource group to contain all testing resources. The resource group name should start with your Microsoft alias. Valid group name examples: `myalias`, `myalias-feature-101-testing`. Your Microsoft account must be [linked to your Github account](https://repos.opensource.microsoft.com/link). diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml index ec015618fad..3616d5cacf2 100644 --- a/eng/pipelines/live-test-cleanup.yml +++ b/eng/pipelines/live-test-cleanup.yml @@ -11,10 +11,11 @@ parameters: - DisplayName: AzureCloud - Resource Cleanup SubscriptionConfigurations: - $(sub-config-azure-cloud-test-resources) - AdditionalParameters: "-DeleteNonCompliantGroups" + AdditionalParameters: "-DeleteNonCompliantGroups -DeleteArmDeployments" - DisplayName: AzureCloud-Preview - Resource Cleanup SubscriptionConfigurations: - $(sub-config-azure-cloud-test-resources-preview) + AdditionalParameters: "-DeleteArmDeployments" # TODO: Enable strict resource cleanup after pre-existing static groups have been handled # AdditionalParameters: "-DeleteNonCompliantGroups" - DisplayName: AzureUSGovernment - Resource Cleanup @@ -30,7 +31,7 @@ parameters: - DisplayName: AzureCloud Playground - Resource Cleanup SubscriptionConfigurations: - $(sub-config-azure-cloud-playground) - AdditionalParameters: "-DeleteNonCompliantGroups" + AdditionalParameters: "-DeleteNonCompliantGroups -DeleteArmDeployments -MaxLifespanDeleteAfterHours 720" - DisplayName: Dogfood Translation - Resource Cleanup SubscriptionConfigurations: - $(sub-config-translation-int-test-resources) diff --git a/eng/scripts/Remove-WormStorageAccounts.ps1 b/eng/scripts/Remove-WormStorageAccounts.ps1 index 6d37365dc39..c0868986655 100644 --- a/eng/scripts/Remove-WormStorageAccounts.ps1 +++ b/eng/scripts/Remove-WormStorageAccounts.ps1 @@ -10,8 +10,6 @@ $ErrorActionPreference = 'Stop' # Be a little defensive so we don't delete non-live test groups via naming convention if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { - Write-Error "The -GroupPrefix parameter must start with 'rg-'" - exit 1 } $groups = Get-AzResourceGroup | ? { $_.ResourceGroupName.StartsWith($GroupPrefix) } | ? { $_.ProvisioningState -ne 'Deleting' } @@ -45,6 +43,8 @@ foreach ($group in $groups) { # and sometimes we must delete the blob if there's a legal hold. # Try to remove the blob, but keep running regardless. try { + Write-Host "Removing immutability policies and blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force } catch {} # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index bdbe25ca313..af7617a034d 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -56,7 +56,10 @@ param ( [switch] $DeleteArmDeployments, [Parameter()] - [int] $DeleteAfterHours = 24, + [Double] $DeleteAfterHours = 24, + + [Parameter()] + [Double] $MaxLifespanDeleteAfterHours, [Parameter()] [string] $AllowListPath = "$PSScriptRoot/cleanup-allowlist.txt", @@ -226,7 +229,7 @@ function HasValidOwnerTag([object]$ResourceGroup) { Write-Warning " Resource group '$($ResourceGroup.ResourceGroupName)' has invalid owner tags: $($invalidOwners -join ',')" } if ($hasValidOwner) { - Write-Host " Skipping tagged resource group '$($ResourceGroup.ResourceGroupName)' with owners '$owners'" + Write-Host " Found tagged resource group '$($ResourceGroup.ResourceGroupName)' with owners '$owners'" return $true } return $false @@ -238,7 +241,7 @@ function HasValidAliasInName([object]$ResourceGroup) { -match '^(rg-)?(?(t-|a-|v-)?[a-z,A-Z]+)([-_].*)?$' ` -and (IsValidAlias -Alias $matches['alias'])) { - Write-Host " Skipping resource group '$($ResourceGroup.ResourceGroupName)' starting with valid alias '$($matches['alias'])'" + Write-Host " Found resource group '$($ResourceGroup.ResourceGroupName)' starting with valid alias '$($matches['alias'])'" return $true } return $false @@ -269,7 +272,8 @@ function HasException([object]$ResourceGroup) { function FindOrCreateDeleteAfterTag { [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param( - [object]$ResourceGroup + [object]$ResourceGroup, + [Double]$HoursToDelete ) if (!$DeleteNonCompliantGroups -or !$ResourceGroup) { @@ -285,7 +289,7 @@ function FindOrCreateDeleteAfterTag { $deleteAfter = GetTag $ResourceGroup "DeleteAfter" if (!$deleteAfter -or !($deleteAfter -as [datetime])) { - $deleteAfter = [datetime]::UtcNow.AddHours($DeleteAfterHours) + $deleteAfter = [datetime]::UtcNow.AddHours($HoursToDelete) if ($Force -or $PSCmdlet.ShouldProcess("$($ResourceGroup.ResourceGroupName) [DeleteAfter (UTC): $deleteAfter]", "Adding DeleteAfter Tag to Group")) { Write-Host "Adding DeleteAfter tag with value '$deleteAfter' to group '$($ResourceGroup.ResourceGroupName)'" $result = ($ResourceGroup | Update-AzTag -Operation Merge -Tag @{ DeleteAfter = $deleteAfter }) 2>&1 @@ -331,10 +335,14 @@ function HasDeleteLock([object]$ResourceGroup) { function DeleteArmDeployments([object]$ResourceGroup) { if (!$DeleteArmDeployments) { - return + return } - Write-Host "Deleting ARM deployments for group $($ResourceGroup.ResourceGroupName) as they may contain secrets. Deployed resources will not be affected." - $null = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Remove-AzResourceGroupDeployment + $toDelete = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Where-Object { $_ -and $_.Outputs?.Count } + if (!$toDelete -or !$toDelete.Count) { + return + } + Write-Host "Deleting $($toDelete.Count) ARM deployments for group $($ResourceGroup.ResourceGroupName) as they may contain output secrets. Deployed resources will not be affected." + $null = $toDelete | Remove-AzResourceGroupDeployment } function DeleteOrUpdateResourceGroups() { @@ -348,8 +356,9 @@ function DeleteOrUpdateResourceGroups() { Write-Verbose "Fetching groups" [Array]$allGroups = Retry { Get-AzResourceGroup } $toDelete = @() - $toUpdate = @() $toClean = @() + $toDeleteSoon = @() + $toDeleteLater = @() Write-Host "Total Resource Groups: $($allGroups.Count)" foreach ($rg in $allGroups) { @@ -366,22 +375,38 @@ function DeleteOrUpdateResourceGroups() { if ((IsChildResource $rg) -or (HasDeleteLock $rg)) { continue } - if ((HasDoNotDeleteTag $rg) -or (HasValidAliasInName $rg) -or (HasValidOwnerTag $rg)) { + if (HasDoNotDeleteTag $rg) { $toClean += $rg continue } - $toUpdate += $rg + if ((HasValidAliasInName $rg) -or (HasValidOwnerTag $rg)) { + $toClean += $rg + $toDeleteLater += $rg + continue + } + + $toDeleteSoon += $rg } - foreach ($rg in $toUpdate) { - FindOrCreateDeleteAfterTag $rg + foreach ($rg in $toDeleteSoon) { + FindOrCreateDeleteAfterTag -ResourceGroup $rg -HoursToDelete $DeleteAfterHours } + if ($MaxLifeSpanDeleteAfterHours) { + foreach ($rg in $toDeleteLater) { + FindOrCreateDeleteAfterTag -ResourceGroup $rg -HoursToDelete $MaxLifespanDeleteAfterHours + } + } + + DeleteAndPurgeGroups $toDelete + foreach ($rg in $toClean) { DeleteArmDeployments $rg } +} +function DeleteAndPurgeGroups([array]$toDelete) { # Get purgeable resources already in a deleted state. $purgeableResources = @(Get-PurgeableResources) From a71de109fdecb572c6f32c39ff01179091e26ce0 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 20 Oct 2023 11:27:33 -0700 Subject: [PATCH 77/93] Added doxygen comment support to C++ API Reviews. (#7167) * Add documentation for C++ types * Correctly handle std::uint8_t * Simpler experience for comment extraction; Don't double dump source comment * clang-format. --- .../ApiViewProcessor/ApiViewMessage.cpp | 17 +- .../ApiViewProcessor/ApiViewMessage.hpp | 1 + .../ApiViewProcessor/ApiViewProcessor.hpp | 1 + .../ApiViewProcessor/AstDumper.cpp | 12 +- .../ApiViewProcessor/AstDumper.hpp | 14 +- .../ApiViewProcessor/AstNode.cpp | 572 ++++++++----- .../ApiViewProcessor/AstNode.hpp | 17 +- .../ApiViewProcessor/CMakeLists.txt | 2 +- .../ApiViewProcessor/CommentExtractor.cpp | 768 ++++++++++++++++++ .../ApiViewProcessor/CommentExtractor.hpp | 70 ++ .../ApiViewProcessor/JsonDumper.hpp | 53 +- .../ApiViewProcessor/ProcessorImpl.cpp | 7 +- .../ApiViewProcessor/ProcessorImpl.hpp | 25 +- .../ApiViewProcessor/TextDumper.hpp | 67 +- .../cpp-api-parser/ParseTests/CMakeLists.txt | 2 +- .../TestCases/DocumentationTests.cpp | 151 ++++ .../cpp-api-parser/ParseTests/tests.cpp | 32 +- .../apiview/parsers/cpp-api-parser/README.md | 2 + 18 files changed, 1555 insertions(+), 258 deletions(-) create mode 100644 tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.cpp create mode 100644 tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.hpp create mode 100644 tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DocumentationTests.cpp diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp index e47ac7586b4..ddbe3702477 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.cpp @@ -87,12 +87,23 @@ void AzureClassesDatabase::CreateApiViewMessage( } case ApiViewMessages::NonVirtualDestructor: { newMessage.DiagnosticId = "CPA000B"; - newMessage.DiagnosticText = "Base class destructors should be public and virtual or protected and non-virtual. "; - newMessage.HelpLinkUri - = "https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#c35-a-base-class-destructor-should-be-either-public-and-virtual-or-protected-and-non-virtual"; + newMessage.DiagnosticText + = "Base class destructors should be public and virtual or protected and non-virtual. "; + newMessage.HelpLinkUri = "https://isocpp.github.io/CppCoreGuidelines/" + "CppCoreGuidelines#c35-a-base-class-destructor-should-be-either-" + "public-and-virtual-or-protected-and-non-virtual"; newMessage.Level = ApiViewMessage::MessageLevel::Error; break; } + + case ApiViewMessages::TypedefInGlobalNamespace: { + newMessage.DiagnosticId = "CPA000C"; + newMessage.DiagnosticText + = "Types in the global namespace which are not builtin types should be avoided. This " + "especially applies to the int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, " + "int64_t, uint64_t types, all of which should be in the std namespace."; + newMessage.Level = ApiViewMessage::MessageLevel::Warning; + } } newMessage.TargetId = targetId; m_diagnostics.push_back(std::move(newMessage)); diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp index 1c3e54d573c..23e25cb0a56 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewMessage.hpp @@ -36,4 +36,5 @@ enum class ApiViewMessages UsingDirectiveFound, // "using namespace" directive found. ImplicitOverride, // Implicit override of virtual method. NonVirtualDestructor, // Destructor of non-final class is not virtual. + TypedefInGlobalNamespace, // A type contains a non-builtin value in the global namespace. }; diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewProcessor.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewProcessor.hpp index 30267679606..fffaa019aaf 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewProcessor.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ApiViewProcessor.hpp @@ -79,6 +79,7 @@ class AzureClassesDatabase { ~AzureClassesDatabase(); TypeHierarchy* GetTypeHierarchy() { return &m_typeHierarchy; } + ApiViewProcessorImpl const* GetProcessor() { return m_processor; } void CreateApiViewMessage(ApiViewMessages diagnostic, std::string_view const& targetId); void CreateAstNode(clang::NamedDecl* namedNode); diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.cpp index b43f4b617e0..4ac053f20fe 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.cpp @@ -135,8 +135,16 @@ void AstDumper::SetNamespace(std::string_view const& newNamespace) } } -void AstDumper::Newline() { InsertNewline(); } +void AstDumper::Newline() +{ + InsertNewline(); + m_currentCursor = 0; +} -void AstDumper::LeftAlign() { InsertWhitespace(m_indentationLevel); } +void AstDumper::LeftAlign() +{ + InsertWhitespace(m_indentationLevel); + m_currentCursor = m_indentationLevel; +} void AstDumper::AdjustIndent(int indentDelta) { m_indentationLevel += indentDelta; } diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.hpp index 0e1df127f08..4b68b06feac 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstDumper.hpp @@ -12,6 +12,7 @@ class AstDumper { std::string m_currentNamespace; std::vector m_namespaceComponents; int m_indentationLevel{}; + size_t m_currentCursor{}; void OpenNamespace(std::string_view const& namespaceName); void OpenNamespaces( @@ -22,6 +23,9 @@ class AstDumper { std::vector const& namespaceComponents, std::vector::iterator& current); +protected: + void UpdateCursor(size_t cursorAdjustment) { m_currentCursor += cursorAdjustment; } + public: // Note about the different functions. // Functions named "InsertXxx" and "AddXxx" are intended to insert elements into the output @@ -30,6 +34,7 @@ class AstDumper { void AdjustIndent(int indentDelta = 0); void LeftAlign(); void Newline(); + size_t GetCurrentCursor() { return m_currentCursor; } void SetNamespace(std::string_view const& currentNamespace); virtual void InsertNewline() = 0; @@ -43,18 +48,21 @@ class AstDumper { std::string_view const& type, std::string_view const& typeNavigationId) = 0; - virtual void InsertMemberName(std::string_view const& member, std::string_view const& memberFullName) = 0; + virtual void InsertMemberName( + std::string_view const& member, + std::string_view const& memberFullName) + = 0; virtual void InsertStringLiteral(std::string_view const& str) = 0; virtual void InsertLiteral(std::string_view const& str) = 0; virtual void InsertComment(std::string_view const& comment) = 0; + virtual void AddExternalLinkStart(std::string_view const& linkValue) = 0; + virtual void AddExternalLinkEnd() = 0; virtual void AddDocumentRangeStart() = 0; virtual void AddDocumentRangeEnd() = 0; virtual void AddDeprecatedRangeStart() = 0; virtual void AddDeprecatedRangeEnd() = 0; virtual void AddDiffRangeStart() = 0; virtual void AddDiffRangeEnd() = 0; - virtual void AddInheritanceInfoStart() = 0; - virtual void AddInheritanceInfoEnd() = 0; virtual void DumpTypeHierarchyNode(std::shared_ptr const& node) = 0; diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp index 89d24e95027..dc043cad33b 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.cpp @@ -2,8 +2,9 @@ // SPDX-License-Identifier: MIT #include "AstNode.hpp" +#include "CommentExtractor.hpp" #include "ProcessorImpl.hpp" -#include +#include "clang/AST/ASTConsumer.h" #include #include #include @@ -38,156 +39,16 @@ std::string AccessSpecifierToString(AccessSpecifier specifier) "Unknown access specifier: " + std::to_string(static_cast(specifier))); } -class MyCommentVisitor : public comments::ConstCommentVisitor { -public: - std::string visitComment(const comments::Comment* comment) - { - std::string rv; - for (auto child = comment->child_begin(); child != comment->child_end(); child++) - { - rv += visit(*child); - } - return rv; - }; - std::string visitFullComment(const comments::FullComment* decl) - { - std::string val; - for (auto child = decl->child_begin(); child != decl->child_end(); child++) - { - val += visit(*child); - } - return val; - }; - std::string visitBlockCommandComment(const comments::BlockCommandComment* bc) - { - // llvm::outs() << "Block command: " - // << comments::CommandTraits::getBuiltinCommandInfo(bc->getCommandID())->Name - // << "\n"; - std::string rv; - if (comments::CommandTraits::getBuiltinCommandInfo(bc->getCommandID())->IsBriefCommand) - { - for (auto child = bc->child_begin(); child != bc->child_end(); child++) - { - rv += visit(*child); - } - } - return rv; - }; - std::string visitHTMLStartTagComment(const comments::HTMLStartTagComment* startTag) - { - std::string rv = "<" + std::string(startTag->getTagName()) + " "; - auto attributeCount = startTag->getNumAttrs(); - for (auto i = 0ul; i < attributeCount; i += 1) - { - auto& attribute{startTag->getAttr(i)}; - rv += std::string(attribute.Name); - rv += "="; - rv += "'" + std::string(attribute.Value) + "'"; - } - rv += ">"; - - return rv; - } - std::string visitHTMLEndTagComment(const comments::HTMLEndTagComment* startTag) - { - std::string rv = "getTagName()) + ">"; - return rv; - } - std::string visitVerbatimBlockComment(const comments::VerbatimBlockComment* vbc) - { - std::string rv; - for (auto child = vbc->child_begin(); child != vbc->child_end(); child++) - { - rv += visit(*child); - } - return rv; - } - std::string visitVerbatimBlockLineComment(const comments::VerbatimBlockLineComment* vbc) - { - std::string rv = std::string(vbc->getText()); - - for (auto child = vbc->child_begin(); child != vbc->child_end(); child++) - { - rv += visit(*child); - } - return rv; - } - std::string visitParagraphComment(const comments::ParagraphComment* decl) - { - std::string rv; - for (auto child = decl->child_begin(); child != decl->child_end(); child++) - { - rv += visit(*child) + "\n"; - } - rv += "\n"; - - return rv; - }; - - std::string visitTextComment(const comments::TextComment* tc) - { - return static_cast(tc->getText()); - }; - - std::string visitHTMLTagComment(const comments::HTMLTagComment* tag) - { - tag->dump(); - return "***HTML Tag Comment***"; - } - std::string visitInlineContentComment(const comments::InlineContentComment* tag) - { - tag->dump(); - return "*** Inline Content Comment ***"; - } - std::string visitInlineCommandComment(const comments::InlineCommandComment* tag) - { - tag->dump(); - return "*** Inline Command Comment ***"; - } - std::string visitParamCommandComment(const comments::ParamCommandComment* tag) - { - tag->dump(); - return "*** Param Command Comment ***"; - } - std::string visitTParamCommandComment(const comments::TParamCommandComment* tag) - { - tag->dump(); - return "*** TParam Command Comment ***"; - } - std::string visitVerbatimLineComment(const comments::VerbatimLineComment* tag) - { - tag->dump(); - return "*** Verbatim Line Comment ***"; - } -}; - -std::string AstNode::GetCommentForNode(ASTContext& context, Decl const* decl) +std::unique_ptr AstNode::GetCommentForNode(ASTContext& context, Decl const* decl) { - auto fullComment{context.getCommentForDecl(decl, nullptr)}; - if (fullComment) - { - MyCommentVisitor commentVisitor; - return commentVisitor.visit(fullComment); - } - return ""; + return ExtractCommentForDeclaration(context, decl); } -std::string AstNode::GetCommentForNode(ASTContext& context, Decl const& decl) -{ - auto fullComment{context.getCommentForDecl(&decl, nullptr)}; - if (fullComment) - { - MyCommentVisitor commentVisitor; - return commentVisitor.visit(fullComment); - } - return ""; -} - -AstNode::AstNode(const Decl*) {} +AstNode::AstNode() {} struct AstTerminalNode : public AstNode { - AstTerminalNode() : AstNode(nullptr) {} + AstTerminalNode() : AstNode() {} void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { @@ -195,8 +56,50 @@ struct AstTerminalNode : public AstNode } }; +/** An AST Type represents a type in the C++ language. + * + */ class AstType { +public: + AstType(QualType type) + : m_isBuiltinType{type->isBuiltinType()}, m_isConstQualified{type.isLocalConstQualified()}, + m_isVolatile{type.isLocalVolatileQualified()}, m_hasQualifiers{type.hasLocalQualifiers()}, + m_isReference{type.getTypePtr()->isReferenceType()}, + m_isRValueReference(type.getTypePtr()->isRValueReferenceType()), + m_isPointer{type.getTypePtr()->isPointerType()} + { + PrintingPolicy pp{LangOptions{}}; + pp.adjustForCPlusPlus(); + m_internalTypeName = QualType::getAsString(type.split(), pp); + m_isInGlobalNamespace = IsTypeInGlobalNamespace(type.getTypePtr()); + } + + AstType(QualType type, const ASTContext& context) + : m_isBuiltinType{type->isBuiltinType()}, m_isConstQualified{type.isLocalConstQualified()}, + m_isVolatile{type.isLocalVolatileQualified()}, m_hasQualifiers{type.hasLocalQualifiers()}, + m_isReference{type.getTypePtr()->isReferenceType()}, + m_isRValueReference(type.getTypePtr()->isRValueReferenceType()), + m_isPointer{type.getTypePtr()->isPointerType()} + { + PrintingPolicy pp{LangOptions{}}; + pp.adjustForCPlusPlus(); + m_internalTypeName = QualType::getAsString(type.split(), pp); + m_isInGlobalNamespace = IsTypeInGlobalNamespace(type.getTypePtr()); + + // Walk the type looking for an inner type which appears to be a reasonable inner type. + // if (typePtr->getTypeClass() != Type::Elaborated && typePtr->getTypeClass() != + // Type::Builtin) + // { + // AstTypeVisitor visitTypes; + // m_underlyingType = visitTypes.Visit(typePtr); + // } + } + void Dump(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const; + + bool IsTypeInGlobalNamespace() const { return m_isInGlobalNamespace; } + +private: struct AstTypeVisitor : public TypeVisitor> { std::unique_ptr VisitQualType(QualType qt) @@ -243,40 +146,145 @@ class AstType { bool m_isReference; bool m_isRValueReference; bool m_isPointer; + bool m_isInGlobalNamespace; /// True if the type references a typedef in the global namespace. std::unique_ptr m_underlyingType; -public: - AstType(QualType type) - : m_isBuiltinType{type->isBuiltinType()}, m_isConstQualified{type.isLocalConstQualified()}, - m_isVolatile{type.isLocalVolatileQualified()}, m_hasQualifiers{type.hasLocalQualifiers()}, - m_isReference{type.getTypePtr()->isReferenceType()}, - m_isRValueReference(type.getTypePtr()->isRValueReferenceType()), - m_isPointer{type.getTypePtr()->isPointerType()} - { - PrintingPolicy pp{LangOptions{}}; - pp.adjustForCPlusPlus(); - m_internalTypeName = QualType::getAsString(type.split(), pp); - } - AstType(QualType type, const ASTContext& context) - : m_isBuiltinType{type->isBuiltinType()}, m_isConstQualified{type.isLocalConstQualified()}, - m_isVolatile{type.isLocalVolatileQualified()}, m_hasQualifiers{type.hasLocalQualifiers()}, - m_isReference{type.getTypePtr()->isReferenceType()}, - m_isRValueReference(type.getTypePtr()->isRValueReferenceType()), - m_isPointer{type.getTypePtr()->isPointerType()} + // Returns true if the type contains a reference to a type which is in the global namespace. + static bool IsTypeInGlobalNamespace(Type const* typePtr) { + // clang TypeVisitor which will iterate over the type and return true if it finds a type which + // is a typedef in the global namespace. + struct IsTypeInGlobalNamespaceVisitor : public TypeVisitor + { + IsTypeInGlobalNamespaceVisitor(){}; + bool VisitTypedefType(TypedefType const* typeDef) + { + clang::PrintingPolicy pp{LangOptions{}}; + pp.adjustForCPlusPlus(); - PrintingPolicy pp{LangOptions{}}; - pp.adjustForCPlusPlus(); - m_internalTypeName = QualType::getAsString(type.split(), pp); - // Walk the type looking for an inner type which appears to be a reasonable inner type. - // if (typePtr->getTypeClass() != Type::Elaborated && typePtr->getTypeClass() != - // Type::Builtin) - // { - // AstTypeVisitor visitTypes; - // m_underlyingType = visitTypes.Visit(typePtr); - // } + auto typeName{QualType::getAsString(QualType(typeDef, 0).split(), pp)}; + if (typeName.find(':') == std::string::npos) + { + // size_t is valid in both the global namespace and std namespace. + if (typeName == "size_t") + { + return false; + } + return true; + } + return false; + } + bool VisitTemplateSpecializationType(TemplateSpecializationType const* tst) + { + for (const auto& arg : tst->template_arguments()) + { + // We only care about type arguments. + if (arg.getKind() == TemplateArgument::ArgKind::Type) + { + if (TypeVisitor::Visit(arg.getAsType().split().Ty)) + { + return true; + } + } + } + return false; + } + bool VisitTemplateArgumentType(TemplateArgument const* ta) + { + return TypeVisitor::Visit(ta->getAsType().split().Ty); + } + bool VisitElaboratedType(ElaboratedType const* et) + { + return TypeVisitor::Visit(et->getNamedType().split().Ty); + } + bool VisitLValueReferenceType(LValueReferenceType const* lrt) + { + return TypeVisitor::Visit(lrt->getPointeeType().split().Ty); + } + bool VisitRValueReferenceType(RValueReferenceType const* rrt) + { + return TypeVisitor::Visit(rrt->getPointeeType().split().Ty); + } + bool VisitPointerType(PointerType const* pt) + { + return TypeVisitor::Visit(pt->getPointeeType().split().Ty); + } + bool VisitPackExpansionType(PackExpansionType const* pe) + { + return TypeVisitor::Visit(pe->getPattern().split().Ty); + } + bool VisitIncompleteArrayType(IncompleteArrayType const* ia) + { + return TypeVisitor::Visit(ia->getArrayElementTypeNoTypeQual()); + } + bool VisitConstantArrayType(ConstantArrayType const* ca) + { + return TypeVisitor::Visit(ca->getArrayElementTypeNoTypeQual()); + } + bool VisitParenType(ParenType const* pt) + { + return TypeVisitor::Visit(pt->getInnerType().split().Ty); + } + // This type was declared with a "using" declaration. + // + bool VisitUsingType(UsingType const* ut) + { + bool rv{TypeVisitor::Visit(ut->getUnderlyingType().split().Ty)}; + // If the underlying type is in the global namespace, but has a shadow declaration (a + // declaration introduced by a using declaration), then we treat it as if it wasn't in the + // global namespace (because it's not). + if (rv) + { + if (ut->getFoundDecl()) + { + return false; + } + } + return rv; + } + + bool VisitFunctionProtoType(FunctionProtoType const* fp) + { + for (const auto& arg : fp->param_types()) + { + if (TypeVisitor::Visit(arg.split().Ty)) + { + return true; + } + } + if (TypeVisitor::Visit(fp->getReturnType().split().Ty)) + { + return true; + } + return false; + } + + // Dependent names don't contain underlying types. + bool VisitDependentNameType(DependentNameType const* dn) { return false; } + // Template type params don't contain underlying types. + bool VisitTemplateTypeParmType(TemplateTypeParmType const* ttp) { return false; } + // An injected class name is a template name referenced without template parameters. + bool VisitInjectedClassNameType(InjectedClassNameType const* ic) { return false; } + // Record type params don't contain underlying types. + bool VisitRecordType(RecordType const* record) { return false; } + // Enum type params don't contain underlying types. + bool VisitEnumType(EnumType const* et) { return false; } + // Builtin type params don't contain underlying types. + bool VisitBuiltinType(BuiltinType const* bt) { return false; } + + bool VisitType(const Type* t) + { + llvm::outs() << "Visit Type " + << QualType::getAsString(QualType(t, 0).split(), LangOptions()) + << "Type class: " << t->getTypeClassName() << "\n"; + t->dump(); + return false; + } + }; + + IsTypeInGlobalNamespaceVisitor visitTypes; + return visitTypes.Visit(typePtr); } - void Dump(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const; }; class AstStatement { @@ -290,7 +298,7 @@ class AstExpr : public AstStatement { protected: AstType m_type; AstExpr(Expr const* expression, ASTContext& context) - : AstStatement(expression, context), m_type{expression->getType()} + : AstStatement(expression, context), m_type{expression->getType(), context} { } @@ -387,7 +395,7 @@ class AstImplicitCastExpr : public AstExpr { public: AstImplicitCastExpr(ImplicitCastExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_underlyingType{expression->getType()}, + : AstExpr(expression, context), m_underlyingType{expression->getType(), context}, m_castValue{AstExpr::Create(*expression->child_begin(), context)} { // Assert that there is a single child of the ImplicitCastExprobject. @@ -404,7 +412,7 @@ class AstCastExpr : public AstExpr { public: AstCastExpr(CastExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_underlyingType{expression->getType()}, + : AstExpr(expression, context), m_underlyingType{expression->getType(), context}, m_castValue{AstExpr::Create(*expression->child_begin(), context)} { // Assert that there is a single child of the ImplicitCastExprobject. @@ -427,7 +435,7 @@ class AstCStyleCastExpr : public AstExpr { public: AstCStyleCastExpr(CStyleCastExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_underlyingType{expression->getType()}, + : AstExpr(expression, context), m_underlyingType{expression->getType(), context}, m_castValue{AstExpr::Create(*expression->child_begin(), context)} { // Assert that there is a single child of the ImplicitCastExprobject. @@ -517,7 +525,7 @@ class AstCtorExpr : public AstExpr { : AstExpr(expression, context) { // Reset the type to the type of the constructor. - m_type = AstType{expression->getType()}; + m_type = AstType{expression->getType(), context}; int argn = 0; for (auto const& arg : expression->arguments()) { @@ -790,7 +798,8 @@ class AstScalarValueInit : public AstExpr { public: AstScalarValueInit(CXXScalarValueInitExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_underlyingType{expression->getTypeSourceInfo()->getType()} + : AstExpr(expression, context), + m_underlyingType{expression->getTypeSourceInfo()->getType(), context} { } @@ -809,7 +818,7 @@ class AstImplicitValueInit : public AstExpr { public: AstImplicitValueInit(ImplicitValueInitExpr const* expression, ASTContext& context) - : AstExpr(expression, context), m_underlyingType{expression->getType()} + : AstExpr(expression, context), m_underlyingType{expression->getType(), context} { } @@ -963,7 +972,7 @@ std::unique_ptr AstExpr::Create(Stmt const* statement, ASTContext& cont class AstAttribute : public AstNode { public: AstAttribute(clang::Attr const* attribute) - : AstNode(nullptr), m_syntax{attribute->getSyntax()}, m_attributeKind{attribute->getKind()}, + : AstNode(), m_syntax{attribute->getSyntax()}, m_attributeKind{attribute->getKind()}, m_attributeName{attribute->getSpelling()} { switch (m_attributeKind) @@ -1106,12 +1115,34 @@ AstNamedNode::AstNamedNode( NamedDecl const* namedDecl, AzureClassesDatabase* const database, std::shared_ptr parentNode) - : AstNode(namedDecl), m_namespace{AstNode::GetNamespaceForDecl(namedDecl)}, + : AstNode(), m_namespace{AstNode::GetNamespaceForDecl(namedDecl)}, m_name{namedDecl->getNameAsString()}, m_classDatabase(database), m_navigationId{namedDecl->getQualifiedNameAsString()}, m_nodeDocumentation{AstNode::GetCommentForNode(namedDecl->getASTContext(), namedDecl)}, m_nodeAccess{namedDecl->getAccess()} { + { + auto location = namedDecl->getLocation(); + auto const& sourceManager = namedDecl->getASTContext().getSourceManager(); + auto const& presumedLocation = sourceManager.getPresumedLoc(location); + std::string typeLocation{presumedLocation.getFilename()}; + + // Remove the root directory from the location if the location is within the root directory. + if (typeLocation.find(database->GetProcessor()->RootDirectory()) == 0) + { + typeLocation.erase(0, database->GetProcessor()->RootDirectory().size() + 1); + } + if (!database->GetProcessor()->SourceRepository().empty()) + { + m_typeUrl = database->GetProcessor()->SourceRepository(); + m_typeUrl += "/" + typeLocation; + m_typeUrl += "#L" + std::to_string(presumedLocation.getLine()); + } + m_typeLocation = typeLocation; + m_typeLocation += ":" + std::to_string(presumedLocation.getLine()); + m_typeLocation += ":" + std::to_string(presumedLocation.getColumn()); + } + if (namedDecl->hasAttrs()) { for (const auto& attr : namedDecl->attrs()) @@ -1146,6 +1177,69 @@ void AstNamedNode::DumpAttributes(AstDumper* dumper, DumpNodeOptions const& opti [](AstDumper* dumper, DumpNodeOptions const& options) { dumper->Newline(); }); } } +void AstNamedNode::DumpDocumentation(AstDumper* dumper, DumpNodeOptions const& options) const +{ + if (options.NeedsDocumentation) + { + if (m_nodeDocumentation) + { + dumper->AddDocumentRangeStart(); + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertComment("/**"); + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = true; + innerOptions.NeedsTrailingNewline = false; + m_nodeDocumentation->DumpNode(dumper, innerOptions); + } + dumper->Newline(); // We need to insert a newline here to ensure that the comment is properly + // closed. + dumper->LeftAlign(); + dumper->InsertComment(" */"); + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + dumper->AddDocumentRangeEnd(); + } + } +} + +// Dump a comment showing where the node is located within the source code. +// If the customer gave us a source URL for the ApiView, include a link to the type. +void AstNamedNode::DumpSourceComment(AstDumper* dumper, DumpNodeOptions const& options) const +{ + if (options.NeedsSourceComment) + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertComment("// "); + if (!m_typeUrl.empty()) + { + dumper->AddExternalLinkStart(m_typeUrl); + dumper->InsertComment(m_typeLocation); + dumper->AddExternalLinkEnd(); + } + else + { + dumper->InsertComment(m_typeLocation); + } + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } +} class AstBaseClass { AstType m_baseClass; @@ -1155,18 +1249,16 @@ class AstBaseClass { AstBaseClass(CXXBaseSpecifier const& base) : m_baseClass{base.getType()}, m_access{base.getAccessSpecifierAsWritten()} {}; - void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions); -}; - -void AstBaseClass::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) -{ - if (m_access != AS_none) + void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) { - dumper->InsertKeyword(AccessSpecifierToString(m_access)); - dumper->InsertWhitespace(); + if (m_access != AS_none) + { + dumper->InsertKeyword(AccessSpecifierToString(m_access)); + dumper->InsertWhitespace(); + } + m_baseClass.Dump(dumper, dumpOptions); } - m_baseClass.Dump(dumper, dumpOptions); -} +}; // For functions, we want the navigation ID to be the full signature, including the return type // to handle overloads. @@ -1240,6 +1332,12 @@ class AstParamVariable : public AstNamedNode { { m_navigationId = GetNavigationId(var); + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_type.IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } + clang::PrintingPolicy pp{LangOptions{}}; pp.adjustForCPlusPlus(); @@ -1330,6 +1428,12 @@ class AstVariable : public AstNamedNode { m_isStatic(var->isStaticDataMember()), m_isConstexpr(var->isConstexpr()), m_isConst(var->getType().isConstQualified()) { + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_type.IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } + clang::PrintingPolicy pp{LangOptions{}}; pp.adjustForCPlusPlus(); m_typeAsString = QualType::getAsString(var->getType().split(), pp); @@ -1355,6 +1459,7 @@ class AstVariable : public AstNamedNode { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { + DumpDocumentation(dumper, dumpOptions); DumpAttributes(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { @@ -1415,6 +1520,12 @@ class AstTemplateParameter : public AstNamedNode { { m_defaultValue = std::make_unique(param->getDefaultArgument(), param->getASTContext()); + + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_defaultValue->IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } } for (auto attr : param->attrs()) @@ -1559,8 +1670,13 @@ class AstNonTypeTemplateParam : public AstNamedNode { std::shared_ptr parentNode) : AstNamedNode(param, database, parentNode), m_defaultArgument(AstExpr::Create(param->getDefaultArgument(), param->getASTContext())), - m_templateType{param->getType()} + m_templateType{param->getType(), param->getASTContext()} { + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_templateType.IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { @@ -1593,9 +1709,15 @@ class AstTypeAlias : public AstNamedNode { TypeAliasDecl const* alias, AzureClassesDatabase* const database, std::shared_ptr parentNode) - : AstNamedNode(alias, database, parentNode), m_aliasedType{alias->getUnderlyingType()} + : AstNamedNode(alias, database, parentNode), + m_aliasedType{alias->getUnderlyingType(), alias->getASTContext()} { + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_aliasedType.IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { @@ -1703,6 +1825,13 @@ class AstFunction : public AstNamedNode { m_exceptionSpecification{func->getExceptionSpecType()} { m_navigationId = GetNavigationId(func); + + // If the type of the return value is in the global namespace, then flag it as an error. + if (m_returnValue.IsTypeInGlobalNamespace()) + { + database->CreateApiViewMessage(ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } + if (m_exceptionSpecification == EST_DependentNoexcept) { auto typePtr = func->getType().getTypePtr(); @@ -1736,6 +1865,7 @@ class AstFunction : public AstNamedNode { { dumper->SetNamespace(Namespace()); } + DumpDocumentation(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -1890,6 +2020,7 @@ class AstMethod : public AstFunction { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { + DumpDocumentation(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -1905,6 +2036,8 @@ class AstMethod : public AstFunction { innerOptions.NeedsLeftAlign = false; innerOptions.NeedsTrailingNewline = false; innerOptions.NeedsTrailingSemi = false; + // We already dumped the documentation for this node, we don't need to do it again. + innerOptions.NeedsDocumentation = false; AstFunction::DumpNode(dumper, innerOptions); } if (m_isConst) @@ -1985,6 +2118,7 @@ class AstConstructor : public AstMethod { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { + DumpDocumentation(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -2000,6 +2134,7 @@ class AstConstructor : public AstMethod { innerOptions.NeedsLeftAlign = false; innerOptions.NeedsTrailingNewline = false; innerOptions.NeedsTrailingSemi = false; + innerOptions.NeedsDocumentation = false; AstMethod::DumpNode(dumper, innerOptions); } if (m_isDefault) @@ -2073,6 +2208,7 @@ class AstDestructor : public AstMethod { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { + DumpDocumentation(dumper, dumpOptions); DumpAttributes(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { @@ -2084,6 +2220,7 @@ class AstDestructor : public AstMethod { innerOptions.NeedsLeftAlign = false; innerOptions.NeedsTrailingNewline = false; innerOptions.NeedsTrailingSemi = false; + innerOptions.NeedsDocumentation = false; AstMethod::DumpNode(dumper, innerOptions); } if (m_isDefault) @@ -2116,10 +2253,10 @@ class AstAccessSpec : public AstNode { public: AstAccessSpec(clang::AccessSpecDecl const* accessSpec, AzureClassesDatabase* const) - : AstNode(accessSpec), m_accessSpecifier{accessSpec->getAccess()} + : AstNode(), m_accessSpecifier{accessSpec->getAccess()} { } - AstAccessSpec(AccessSpecifier specifier) : AstNode(nullptr), m_accessSpecifier{specifier} {} + AstAccessSpec(AccessSpecifier specifier) : AstNode(), m_accessSpecifier{specifier} {} void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { // We want to left-indent the "public:", "private:" and "protected" items so they stick @@ -2214,12 +2351,15 @@ class AstClassTemplate : public AstNamedNode { void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { - DumpAttributes(dumper, dumpOptions); if (!Namespace().empty()) { dumper->SetNamespace(Namespace()); } + DumpSourceComment(dumper, dumpOptions); + DumpDocumentation(dumper, dumpOptions); + DumpAttributes(dumper, dumpOptions); + if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -2230,6 +2370,8 @@ class AstClassTemplate : public AstNamedNode { { DumpNodeOptions innerOptions{dumpOptions}; innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsSourceComment + = false; // We've already dumped the source comment for this node. DumpList( m_parameters.begin(), @@ -2247,6 +2389,8 @@ class AstClassTemplate : public AstNamedNode { DumpNodeOptions innerOptions{dumpOptions}; innerOptions.NeedsLeftAlign = true; innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsSourceComment + = false; // We've already dumped the source comment for this node. m_templateBody->DumpNode(dumper, innerOptions); } } @@ -2272,7 +2416,6 @@ class AstFunctionTemplate : public AstNamedNode { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { - DumpAttributes(dumper, dumpOptions); if (!Namespace().empty()) { if (dumpOptions.NeedsNamespaceAdjustment) @@ -2281,6 +2424,9 @@ class AstFunctionTemplate : public AstNamedNode { } } + DumpDocumentation(dumper, dumpOptions); + DumpAttributes(dumper, dumpOptions); + if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -2330,7 +2476,6 @@ class AstTypeAliasTemplate : public AstNamedNode { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { - DumpAttributes(dumper, dumpOptions); if (!Namespace().empty()) { if (dumpOptions.NeedsNamespaceAdjustment) @@ -2339,6 +2484,9 @@ class AstTypeAliasTemplate : public AstNamedNode { } } + DumpDocumentation(dumper, dumpOptions); + DumpAttributes(dumper, dumpOptions); + if (dumpOptions.NeedsLeftAlign) { dumper->LeftAlign(); @@ -2478,13 +2626,19 @@ class AstField : public AstNamedNode { AzureClassesDatabase* const azureClassesDatabase, std::shared_ptr parentNode) : AstNamedNode(fieldDecl, azureClassesDatabase, parentNode), - m_fieldType{fieldDecl->getType()}, + m_fieldType{fieldDecl->getType(), fieldDecl->getASTContext()}, m_initializer{ AstExpr::Create(fieldDecl->getInClassInitializer(), fieldDecl->getASTContext())}, m_classInitializerStyle{fieldDecl->getInClassInitStyle()}, m_hasDefaultMemberInitializer{fieldDecl->hasInClassInitializer()}, m_isMutable{fieldDecl->isMutable()}, m_isConst{fieldDecl->getType().isConstQualified()} { + // If the type of the parameter is in the global namespace, then flag it as an error. + if (m_fieldType.IsTypeInGlobalNamespace()) + { + azureClassesDatabase->CreateApiViewMessage( + ApiViewMessages::TypedefInGlobalNamespace, m_navigationId); + } } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override; }; @@ -2498,7 +2652,7 @@ class AstFriend : public AstNode { FriendDecl const* friendDecl, AzureClassesDatabase* const azureClassesDatabase, std::shared_ptr parentNode) - : AstNode(friendDecl) + : AstNode() { if (friendDecl->getFriendType()) { @@ -2551,9 +2705,8 @@ class AstUsingDirective : public AstNode { UsingDirectiveDecl const* usingDirective, AzureClassesDatabase* const azureClassesDatabase, std::shared_ptr parentNode) - : AstNode(usingDirective), - m_namedNamespace{ - usingDirective->getNominatedNamespaceAsWritten()->getQualifiedNameAsString()} + : AstNode(), m_namedNamespace{ + usingDirective->getNominatedNamespaceAsWritten()->getQualifiedNameAsString()} { azureClassesDatabase->CreateApiViewMessage( ApiViewMessages::UsingDirectiveFound, m_namedNamespace); @@ -2641,6 +2794,7 @@ class AstEnumerator : public AstNamedNode { } void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { + DumpDocumentation(dumper, dumpOptions); DumpAttributes(dumper, dumpOptions); dumper->LeftAlign(); dumper->InsertMemberName(Name(), m_navigationId); @@ -2937,7 +3091,6 @@ AstClassLike::AstClassLike( void AstClassLike::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const { - DumpAttributes(dumper, dumpOptions); if (!Namespace().empty()) { if (dumpOptions.NeedsNamespaceAdjustment) @@ -2945,6 +3098,9 @@ void AstClassLike::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOption dumper->SetNamespace(Namespace()); } } + DumpSourceComment(dumper, dumpOptions); + DumpDocumentation(dumper, dumpOptions); + DumpAttributes(dumper, dumpOptions); // If we're a templated class, don't insert the extra newline before the class // definition. @@ -3034,6 +3190,9 @@ void AstEnum::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) co dumper->SetNamespace(Namespace()); } } + + DumpSourceComment(dumper, dumpOptions); + DumpDocumentation(dumper, dumpOptions); DumpAttributes(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { @@ -3103,6 +3262,7 @@ void AstEnum::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) co void AstField::DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const { + DumpDocumentation(dumper, dumpOptions); DumpAttributes(dumper, dumpOptions); if (dumpOptions.NeedsLeftAlign) { diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.hpp index 6634c70a606..4e1b195b319 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/AstNode.hpp @@ -18,9 +18,13 @@ class Attr; enum AccessSpecifier : int; } // namespace clang +class AstDocumentation; + struct DumpNodeOptions { bool DumpListInitializer{false}; + bool NeedsDocumentation{true}; + bool NeedsSourceComment{true}; bool NeedsLeftAlign{true}; bool NeedsLeadingNewline{true}; bool NeedsTrailingNewline{true}; @@ -28,11 +32,13 @@ struct DumpNodeOptions bool NeedsNamespaceAdjustment{true}; bool IncludeNamespace{false}; bool IncludeContainingClass{false}; + bool InlineBlockComment{false}; + size_t RightMargin{80}; // Soft right margin for dumper. }; class AstNode { protected: - explicit AstNode(clang::Decl const* decl); + explicit AstNode(); public: // AstNode's don't have namespaces or names, so return something that would make callers happy. @@ -41,8 +47,7 @@ class AstNode { virtual void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const = 0; - static std::string GetCommentForNode(clang::ASTContext& context, clang::Decl const* decl); - static std::string GetCommentForNode(clang::ASTContext& context, clang::Decl const& decl); + static std::unique_ptr GetCommentForNode(clang::ASTContext& context, clang::Decl const* decl); static std::unique_ptr Create( clang::Decl const* decl, AzureClassesDatabase* const azureClassesDatabase, @@ -54,12 +59,14 @@ class AstNode { class AstNamedNode : public AstNode { std::string m_namespace; std::string m_name; + std::string m_typeUrl; + std::string m_typeLocation; std::vector> m_nodeAttributes; protected: AzureClassesDatabase* const m_classDatabase; std::string m_navigationId; - std::string m_nodeDocumentation; + std::unique_ptr m_nodeDocumentation; clang::AccessSpecifier m_nodeAccess; explicit AstNamedNode( @@ -69,6 +76,8 @@ class AstNamedNode : public AstNode { public: void DumpAttributes(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const; + void DumpDocumentation(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const; + void DumpSourceComment(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const; virtual void DumpNode(AstDumper* dumper, DumpNodeOptions const& dumpOptions) const override { assert(!"Pure virtual base - missing implementation of DumpNode in derived class."); diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CMakeLists.txt b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CMakeLists.txt index 24d808bb752..5695b0e87d0 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CMakeLists.txt +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CMakeLists.txt @@ -22,7 +22,7 @@ include(HandleLLVMOptions) add_definitions(${LLVM_DEFINITIONS}) add_compile_options(/EHsc) -add_library(ApiViewProcessor STATIC AstNode.cpp ApiViewProcessor.cpp ProcessorImpl.cpp AstDumper.cpp ApiViewMessage.cpp) +add_library(ApiViewProcessor STATIC AstNode.cpp ApiViewProcessor.cpp ProcessorImpl.cpp AstDumper.cpp ApiViewMessage.cpp CommentExtractor.cpp) target_include_directories(ApiViewProcessor PRIVATE ${LLVM_INCLUDE_DIRS}) diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.cpp new file mode 100644 index 00000000000..d7a04612589 --- /dev/null +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.cpp @@ -0,0 +1,768 @@ +// Co pyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "CommentExtractor.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace clang; + +struct AstComment : public AstDocumentation +{ + AstComment(comments::FullComment const* const comment) : AstDocumentation() {} + + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + for (auto& child : m_children) + { + if (child) + { + child->DumpNode(dumper, options); + } + } + } +}; + +struct AstBlockCommandComment : public AstDocumentation +{ + AstBlockCommandComment(comments::BlockCommandComment const* const comment) : AstDocumentation() + { + std::string value; + value += GetCommandMarker(comment->getCommandMarker()); + auto commandInfo{ + clang::comments::CommandTraits::getBuiltinCommandInfo(comment->getCommandID())}; + if (commandInfo->IsBriefCommand) + { + value += "brief"; + } + else if (commandInfo->IsReturnsCommand) + { + value += "returns"; + } + else if (commandInfo->IsThrowsCommand) + { + value += "throws"; + } + else if (commandInfo->IsParamCommand) + { + throw std::runtime_error("Block command comment should never have a param command."); + } + else if (commandInfo->IsTParamCommand) + { + throw std::runtime_error("Block command comment should never have a tparam command."); + } + else if (commandInfo->IsVerbatimBlockCommand) + { + throw std::runtime_error("Block command comment should never have a verbatim command."); + } + else if (commandInfo->IsVerbatimLineCommand) + { + throw std::runtime_error("Block command comment should never have a verbatim line command."); + } + else + { + value += commandInfo->Name; + } + m_thisLine = value; + } + + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + dumper->InsertComment(m_thisLine); + + // The first child will be the first line of the brief description, it should be joined with the + // current line. + auto child{m_children.begin()}; + if (child != m_children.end() && *child) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = false; + innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsTrailingNewline = true; + innerOptions.InlineBlockComment = true; + (*child)->DumpNode(dumper, innerOptions); + child++; + } + + for (; child != m_children.end(); child++) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = true; + innerOptions.NeedsTrailingNewline = false; + if (*child) + { + (*child)->DumpNode(dumper, options); + } + } + + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } +}; + +struct AstParamComment : public AstDocumentation +{ + AstParamComment(comments::ParamCommandComment const* comment) : AstDocumentation() + { + std::string thisLine; + thisLine += GetCommandMarker(comment->getCommandMarker()); + auto commandInfo{ + clang::comments::CommandTraits::getBuiltinCommandInfo(comment->getCommandID())}; + if (commandInfo->IsParamCommand) + { + thisLine += "param"; + } + else + { + thisLine += commandInfo->Name; + } + thisLine += " "; + // If the caller explicitly listed the direction, include that in the description. + if (comment->isDirectionExplicit()) + { + thisLine += comment->getDirectionAsString(comment->getDirection()); + thisLine += " "; + } + if (comment->hasParamName()) + { + thisLine += comment->getParamNameAsWritten(); + thisLine += " "; + } + m_thisLine = thisLine; + } + + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (m_thisLine == "@param format") + { + std::cout << "@param[in] format"; + } + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + dumper->InsertComment(m_thisLine); + + // The first child will be the first line of the parameter documentation, it should be joined + // with the current line. + auto child{m_children.begin()}; + if (child != m_children.end() && *child) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = false; + innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsTrailingNewline = true; + innerOptions.InlineBlockComment = true; + (*child)->DumpNode(dumper, innerOptions); + child++; + } + + for (; child != m_children.end(); child++) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsTrailingNewline = true; + if (*child) + { + (*child)->DumpNode(dumper, options); + } + } + + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } +}; + +struct AstTParamComment : public AstDocumentation +{ + AstTParamComment(comments::TParamCommandComment const* comment) : AstDocumentation() + { + std::string thisLine; + thisLine += GetCommandMarker(comment->getCommandMarker()); + auto commandInfo{ + clang::comments::CommandTraits::getBuiltinCommandInfo(comment->getCommandID())}; + if (commandInfo->IsTParamCommand) + { + thisLine += "tparam"; + } + else + { + thisLine += commandInfo->Name; + } + thisLine += " "; + + if (comment->hasParamName()) + { + thisLine += comment->getParamNameAsWritten(); + thisLine += " "; + } + m_thisLine = thisLine; + } + bool IsInlineComment() const override { return false; } + + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + dumper->InsertComment(m_thisLine); + + // The first child will be the first line of the parameter documentation, it should be joined + // with the current line. + auto child{m_children.begin()}; + if (child != m_children.end() && *child) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = false; + innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsTrailingNewline = true; + innerOptions.InlineBlockComment = true; + (*child)->DumpNode(dumper, innerOptions); + child++; + } + + for (; child != m_children.end(); child++) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = true; + innerOptions.NeedsTrailingNewline = false; + if (*child) + { + (*child)->DumpNode(dumper, options); + } + } + + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } +}; + +struct AstVerbatimBlockComment : public AstDocumentation +{ + AstVerbatimBlockComment(comments::VerbatimBlockComment const* comment) : AstDocumentation() + { + std::string thisLine; + auto commandInfo{ + clang::comments::CommandTraits::getBuiltinCommandInfo(comment->getCommandID())}; + + thisLine += GetCommandMarker(comment->getCommandMarker()); + thisLine += commandInfo->Name; + + auto it = comment->child_begin(); + auto childLineComment = clang::dyn_cast(*it); + if (childLineComment) + { + std::string childText{childLineComment->getText()}; + // If the first character of the 0th argument is a '{', then this is a code block. Append + // it to the name. + if ((childText[0] == '{') && (childText[(childText.size() - 1)] = '}')) + { + m_hasLanguageTag = true; + } + } + m_thisLine = thisLine; + m_endMarker += GetCommandMarker(comment->getCommandMarker()); + m_endMarker += commandInfo->EndCommandName; + } + + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertComment("* "); + dumper->InsertWhitespace(); + dumper->InsertComment(m_thisLine); + + // The first child will be the first line of the parameter documentation, it should be joined + // with the current line. + auto child{m_children.begin()}; + if (child != m_children.end() && *child) + { + DumpNodeOptions innerOptions{options}; + if (m_hasLanguageTag) + { + innerOptions.NeedsLeftAlign = false; + innerOptions.NeedsLeadingNewline = false; + innerOptions.NeedsTrailingNewline = false; + innerOptions.InlineBlockComment = true; + } + (*child)->DumpNode(dumper, innerOptions); + child++; + } + + for (; child != m_children.end(); child++) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = true; + innerOptions.NeedsTrailingNewline = false; + if (*child) + { + (*child)->DumpNode(dumper, options); + } + } + + if (!m_endMarker.empty()) + { + dumper->Newline(); + + dumper->LeftAlign(); + dumper->InsertWhitespace(); + dumper->InsertComment("* "); + dumper->InsertWhitespace(); + dumper->InsertComment(m_endMarker); + } + + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } + +private: + bool m_hasLanguageTag{false}; + std::string m_endMarker; +}; + +// Represents an inline command marker. Examples include the \c in \c foo, or the \a in \a foo. +// The marker is the \c or \a. +// +// \p or \c should be rendered in a fixed width font +// \a or \e or \em should be rendered in a italic font +// \b should be rendered in a bold font +// \emoji should be rendered as an emoji (if possible - see +// https://gist.github.com/rxaviers/7360908). +struct AstInlineCommand : AstDocumentation +{ + AstInlineCommand(const comments::InlineCommandComment* comment) : AstDocumentation() + { + std::string thisLine; + std::string commandRenderMarkdownStart; + std::string commandRenderMarkdownEnd; + switch (comment->getRenderKind()) + { + case clang::comments::InlineCommandComment::RenderKind::RenderNormal: + break; + case clang::comments::InlineCommandComment::RenderKind::RenderBold: + commandRenderMarkdownEnd = "**"; + commandRenderMarkdownStart = "**"; + break; + case clang::comments::InlineCommandComment::RenderKind::RenderEmphasized: + commandRenderMarkdownEnd = "*"; + commandRenderMarkdownStart = "*"; + break; + case clang::comments::InlineCommandComment::RenderKind::RenderMonospaced: + commandRenderMarkdownEnd = "`"; + commandRenderMarkdownStart = "`"; + break; + default: + throw std::runtime_error("Unknown inline command render kind."); + } + // Include the arguments to the command. + thisLine += commandRenderMarkdownStart; + for (unsigned i = 0u; i < comment->getNumArgs(); ++i) + { + thisLine += comment->getArgText(i); + } + thisLine += commandRenderMarkdownEnd; + m_thisLine = thisLine; + } + bool IsInlineComment() const override { return true; } + + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + dumper->InsertComment(m_thisLine); + for (auto& child : m_children) + { + child->DumpNode(dumper, options); + } + } +}; + +// A paragraph represents a block of text. Typically this is a paragraph of text. The children of +// the line are typically AstTextComment nodes, but they may also be AstInlineCommand nodes. If they +// are AstInlineCommand nodes, we should just insert them with no separation, if they are +// AstTextComment nodes, we should insert them with a new line and comment leader between them. +struct AstParagraphComment : AstDocumentation +{ + AstParagraphComment(comments::ParagraphComment const* comment) : AstDocumentation() {} + bool IsInlineComment() const override { return false; } + + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (!options.InlineBlockComment) + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + + // Insert a blank line before the paragraph if the previous line was not an inline comment. + dumper->Newline(); + dumper->LeftAlign(); + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + } + bool insertLineBreak = false; + for (auto& child : m_children) + { + if (child) + { + if (insertLineBreak && !child->IsInlineComment()) + { + dumper->Newline(); + dumper->LeftAlign(); + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + } + child->DumpNode(dumper, options); + if (child->IsInlineComment()) + { + insertLineBreak = false; + } + else + { + insertLineBreak = true; + } + } + } + } +}; + +struct AstVerbatimBlockLineComment : AstDocumentation +{ + AstVerbatimBlockLineComment(comments::VerbatimBlockLineComment const* comment) + : AstDocumentation() + { + m_thisLine = comment->getText(); + } + + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + if (!options.InlineBlockComment) + { + dumper->InsertWhitespace(); + dumper->InsertPunctuation('*'); + dumper->InsertWhitespace(); + } + dumper->InsertComment(m_thisLine); + for (auto& child : m_children) + { + child->DumpNode(dumper, options); + } + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } +}; + +struct AstTextComment : AstDocumentation +{ + AstTextComment(comments::TextComment const* comment) : AstDocumentation() + { + m_thisLine = comment->getText(); + } + bool IsInlineComment() const override { return false; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + dumper->InsertComment(m_thisLine); + } +}; + +struct AstVerbatimLineComment : AstDocumentation +{ + AstVerbatimLineComment(comments::VerbatimLineComment const* comment) : AstDocumentation() + { + auto commandInfo{ + clang::comments::CommandTraits::getBuiltinCommandInfo(comment->getCommandID())}; + m_thisLine += GetCommandMarker(comment->getCommandMarker()); + m_thisLine += commandInfo->Name; + m_endMarker = commandInfo->EndCommandName; + } + bool IsInlineComment() const override { return true; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + dumper->InsertComment(m_thisLine); + } + +private: + std::string m_endMarker; +}; + +struct AstHtmlStartTagComment : AstDocumentation +{ + AstHtmlStartTagComment(comments::HTMLStartTagComment const* comment) : AstDocumentation() + { + if (comment->getTagName() == "a") + { + m_isLinkHref = true; + auto argCount = comment->getNumAttrs(); + for (size_t i = 0; i < argCount; i += 1) + { + if (comment->getAttr(i).Name == "href") + { + m_linkTarget = comment->getAttr(i).Value; + } + } + } + } + bool IsInlineComment() const override { return true; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + // We only serialize the first link argument. + if (m_isLinkHref) + { + dumper->AddExternalLinkStart(m_linkTarget); + } + } + +private: + std::string m_linkTarget; + bool m_isLinkHref{false}; +}; + +struct AstHtmlEndTagComment : AstDocumentation +{ + AstHtmlEndTagComment(comments::HTMLEndTagComment const* comment) : AstDocumentation() + { + if (comment->getTagName() == "a") + { + m_isLinkHref = true; + } + } + bool IsInlineComment() const override { return true; } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (m_isLinkHref) + { + dumper->AddExternalLinkEnd(); + } + } + +private: + bool m_isLinkHref{false}; +}; + +std::unique_ptr AstDocumentation::Create(const comments::Comment* comment) +{ + switch (comment->getCommentKind()) + { + case comments::Comment::CommentKind::FullCommentKind: + return std::make_unique(cast(comment)); + case comments::Comment::CommentKind::BlockCommandCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::CommentKind::ParamCommandCommentKind: + return std::make_unique(cast(comment)); + case comments::Comment::CommentKind::TParamCommandCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::CommentKind::VerbatimBlockCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::CommentKind::InlineCommandCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::ParagraphCommentKind: + return std::make_unique(cast(comment)); + case comments::Comment::TextCommentKind: + return std::make_unique(cast(comment)); + case comments::Comment::VerbatimBlockLineCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::VerbatimLineCommentKind: + return std::make_unique( + cast(comment)); + + case comments::Comment::HTMLStartTagCommentKind: + return std::make_unique( + cast(comment)); + case comments::Comment::HTMLEndTagCommentKind: + return std::make_unique( + cast(comment)); + + default: + llvm::errs() << "Unknown comment kind: " << comment->getCommentKindName() << "\n"; + return nullptr; + } +} + +// clang visitor to extract comments from the AST. +// +// clang comment visitors look for methods named "visitComment". If the method is found, it is +// called, otherwise the comment visitor tries the parent type of the comment. This allows us to +// specialize the visitor for different types of comments but leaves processing for most comments +// inside the visitComment method. +class CommentVisitor + : public clang::comments::CommentVisitor> { +public: + CommentVisitor() + : clang::comments::CommentVisitor>() + { + } + + // Primary processor for comments. This method is called for all comments which do not have a + // specialized visitor. + std::unique_ptr visitComment(const clang::comments::Comment* comment) + { + std::unique_ptr rv{AstDocumentation::Create(comment)}; + for (auto child = comment->child_begin(); child != comment->child_end(); child++) + { + auto childNode = visit(*child); + if (childNode) + { + rv->AddChild(std::move(childNode)); + } + } + return rv; + }; + + // Process a full comment. This is the top level comment type. + std::unique_ptr visitFullComment(const clang::comments::FullComment* decl) + { + // decl->dump(llvm::outs(), m_context); + std::unique_ptr rv{AstDocumentation::Create(decl)}; + for (auto child = decl->child_begin(); child != decl->child_end(); child++) + { + auto childNode = visit(*child); + if (childNode) + { + rv->AddChild(std::move(childNode)); + } + } + return rv; + }; + + // We want to ignore empty paragraph comments, so we need to specialize the visitor for paragraph + // comments. + std::unique_ptr visitParagraphComment( + const clang::comments::ParagraphComment* decl) + { + // Ignore empty paragraph clang::comments. + if (decl->isWhitespace()) + { + return nullptr; + } + std::unique_ptr node{AstDocumentation::Create(decl)}; + for (auto child = decl->child_begin(); child != decl->child_end(); child++) + { + auto childNode = visit(*child); + if (childNode) + { + node->AddChild(std::move(childNode)); + } + } + + return node; + }; + + // We want to ignore empty text comments, so we need to specialize the visitor for text + // comments. + std::unique_ptr visitTextComment(const clang::comments::TextComment* tc) + { + // Ignore text clang::comments which are whitespace. + if (tc->isWhitespace()) + { + return nullptr; + } + std::unique_ptr node{AstDocumentation::Create(tc)}; + return node; + }; +}; + +// Use a commentVisitor to extract all the comments from a comment node. +std::unique_ptr ExtractCommentForDeclaration( + clang::ASTContext const& context, + clang::Decl const* decl) +{ + auto comment = context.getCommentForDecl(decl, nullptr); + if (comment != nullptr) + { + CommentVisitor visitor; + std::unique_ptr doc{visitor.visit(comment)}; + return doc; + } + return nullptr; +} + +std::string_view AstDocumentation::GetCommandMarker(clang::comments::CommandMarkerKind marker) +{ + switch (marker) + { + case clang::comments::CommandMarkerKind::CMK_At: + return "@"; + case clang::comments::CommandMarkerKind::CMK_Backslash: + return "\\"; + } + throw std::runtime_error("Unknown command marker kind."); +} diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.hpp new file mode 100644 index 00000000000..fa335a6ba29 --- /dev/null +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/CommentExtractor.hpp @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "AstDumper.hpp" +#include "AstNode.hpp" + +namespace clang { namespace comments { + + class Comment; + enum CommandMarkerKind : int; +}} // namespace clang::comments + +/** An AstDocumentation node represents a parsed comment. + * It is a base class which will be + * specialized for different types of comments, loosely following the clang AST for comments. + */ +class AstDocumentation : public AstNode { +public: + virtual bool IsInlineComment() const = 0; + void AddChild(std::unique_ptr&& line) { m_children.push_back(std::move(line)); } + void DumpNode(AstDumper* dumper, DumpNodeOptions const& options) const override + { + if (options.NeedsLeadingNewline) + { + dumper->Newline(); + } + if (options.NeedsLeftAlign) + { + dumper->LeftAlign(); + } + dumper->InsertWhitespace(); + dumper->InsertComment("* "); + dumper->InsertWhitespace(); + dumper->InsertComment(m_thisLine); + + for (auto const& child : m_children) + { + DumpNodeOptions innerOptions{options}; + innerOptions.NeedsLeftAlign = true; + innerOptions.NeedsLeadingNewline = true; + innerOptions.NeedsTrailingNewline = false; + if (child) + { + child->DumpNode(dumper, options); + } + } + + if (options.NeedsTrailingNewline) + { + dumper->Newline(); + } + } + + static std::unique_ptr Create(clang::comments::Comment const* comment); + +protected: + AstDocumentation() : AstNode(){}; + std::string_view GetCommandMarker(clang::comments::CommandMarkerKind marker); + + std::vector> m_children; + std::string m_thisLine{}; +}; + +/** Extract a comment from a comment node. + * This function iterates over a clang::comments::Comment + * node and retrieves all the information in the comment in a way which can later be dumped. + */ +std::unique_ptr ExtractCommentForDeclaration( + clang::ASTContext const& context, + clang::Decl const* declaration); diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/JsonDumper.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/JsonDumper.hpp index 71e1a49ff10..fa65ea9dece 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/JsonDumper.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/JsonDumper.hpp @@ -34,8 +34,21 @@ class JsonDumper : public AstDumper { DeprecatedRangeEnd = 14, SkipDiffRangeStart = 15, SkipDiffRangeEnd = 16, - InheritanceInfoStart = 17, - InheritanceInfoEnd = 18 + FoldableSectionHeading = 17, + FoldableSectionContentStart = 18, + FoldableSectionContentEnd = 19, + TableBegin = 20, + TableEnd = 21, + TableRowCount = 22, + TableColumnCount = 23, + TableColumnName = 24, + TableCellBegin = 25, + TableCellEnd = 26, + LeafSectionPlaceholder = 27, + ExternalLinkStart = 28, + ExternalLinkEnd = 29, + HiddenApiRangeStart = 30, + HiddenApiRangeEnd = 31 }; // Validate that the json we've created won't cause problems for ApiView. @@ -105,6 +118,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", whiteSpace}, {"Kind", TokenKinds::Whitespace}}); + UpdateCursor(count); } virtual void InsertNewline() override { @@ -121,6 +135,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", keyword}, {"Kind", TokenKinds::Keyword}}); + UpdateCursor(keyword.size()); } virtual void InsertText(std::string_view const& text) override { @@ -129,6 +144,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", text}, {"Kind", TokenKinds::Text}}); + UpdateCursor(text.size()); } virtual void InsertPunctuation(const char punctuation) override { @@ -138,6 +154,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", punctuationString}, {"Kind", TokenKinds::Punctuation}}); + UpdateCursor(1); } virtual void InsertLineIdMarker() override { @@ -154,6 +171,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", id}, {"Kind", TokenKinds::TypeName}}); + UpdateCursor(id.size()); } virtual void InsertTypeName(std::string_view const& type, std::string_view const& navigationId) override @@ -163,6 +181,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", navigationId}, {"Value", type}, {"Kind", TokenKinds::TypeName}}); + UpdateCursor(type.size()); } virtual void InsertMemberName( std::string_view const& member, @@ -181,6 +200,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", str}, {"Kind", TokenKinds::StringLiteral}}); + UpdateCursor(str.size()); } virtual void InsertLiteral(std::string_view const& str) override { @@ -189,6 +209,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", str}, {"Kind", TokenKinds::Literal}}); + UpdateCursor(str.size()); } virtual void InsertComment(std::string_view const& comment) override { @@ -197,6 +218,7 @@ class JsonDumper : public AstDumper { {"NavigateToId", nullptr}, {"Value", comment}, {"Kind", TokenKinds::Comment}}); + UpdateCursor(comment.size()); } virtual void AddDocumentRangeStart() override { @@ -214,53 +236,54 @@ class JsonDumper : public AstDumper { {"Value", nullptr}, {"Kind", TokenKinds::DocumentRangeEnd}}); } - virtual void AddDeprecatedRangeStart() override + virtual void AddExternalLinkStart(const std::string_view& url) override { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, - {"Value", nullptr}, - {"Kind", TokenKinds::DeprecatedRangeStart}}); + {"Value", url}, + {"Kind", TokenKinds::ExternalLinkEnd}}); + } - virtual void AddDeprecatedRangeEnd() override + virtual void AddExternalLinkEnd() { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, {"Value", nullptr}, - {"Kind", TokenKinds::DeprecatedRangeEnd}}); + {"Kind", TokenKinds::ExternalLinkStart}}); } - virtual void AddDiffRangeStart() override + virtual void AddDeprecatedRangeStart() override { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, {"Value", nullptr}, - {"Kind", TokenKinds::SkipDiffRangeStart}}); + {"Kind", TokenKinds::DeprecatedRangeStart}}); } - virtual void AddDiffRangeEnd() override + virtual void AddDeprecatedRangeEnd() override { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, {"Value", nullptr}, - {"Kind", TokenKinds::SkipDiffRangeEnd}}); + {"Kind", TokenKinds::DeprecatedRangeEnd}}); } - virtual void AddInheritanceInfoStart() override + virtual void AddDiffRangeStart() override { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, {"Value", nullptr}, - {"Kind", TokenKinds::InheritanceInfoStart}}); + {"Kind", TokenKinds::SkipDiffRangeStart}}); } - virtual void AddInheritanceInfoEnd() override + virtual void AddDiffRangeEnd() override { m_json["Tokens"].push_back( {{"DefinitionId", nullptr}, {"NavigateToId", nullptr}, {"Value", nullptr}, - {"Kind", TokenKinds::InheritanceInfoEnd}}); + {"Kind", TokenKinds::SkipDiffRangeEnd}}); } nlohmann::json DoDumpTypeHierarchyNode( diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp index 5dd1cb16b1e..107e5de4e44 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.cpp @@ -101,6 +101,7 @@ const std::vector KnownSettings{ "filterNamespace", "additionalCompilerSwitches", "additionalIncludeDirectories", + "sourceRootUrl", "reviewName", "serviceName", "packageName", @@ -139,6 +140,10 @@ ApiViewProcessorImpl::ApiViewProcessorImpl( { m_includePrivate = configurationJson["includePrivate"]; } + if (configurationJson.contains("sourceRootUrl")) + { + m_repositoryRoot = configurationJson["sourceRootUrl"]; + } if (configurationJson.contains("filterNamespace") && !configurationJson["filterNamespace"].is_null()) { @@ -430,7 +435,7 @@ int ApiViewProcessorImpl::ProcessApiView() assert(file.u8string().find(m_currentSourceRoot.u8string()) == 0); auto relativeFile = static_cast( stringFromU8string(file.u8string().erase(0, m_currentSourceRoot.u8string().size() + 1))); - std::string quotedFile = replaceAll(relativeFile, "\\", "\\\\"); + std::string quotedFile = replaceAll(relativeFile, "\\", "/"); sourceFileAggregate << "#include \"" << quotedFile << "\"" << std::endl; } diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.hpp index bfb885778c1..bca4654c28d 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/ProcessorImpl.hpp @@ -27,6 +27,8 @@ class ApiViewProcessorImpl { std::string m_reviewName; std::string m_serviceName; std::string m_packageName; + std::string m_repositoryRoot; + mutable std::string m_sourceRoot; bool m_allowInternal{false}; bool m_includeDetail{false}; @@ -114,11 +116,20 @@ class ApiViewProcessorImpl { std::unique_ptr const& GetClassesDatabase() { return m_classDatabase; } - bool AllowInternal() { return m_allowInternal; } - bool IncludeDetail() { return m_includeDetail; } - bool IncludePrivate() { return m_includePrivate; } - std::string_view const ReviewName() { return m_reviewName; }; - std::string_view const ServiceName() { return m_serviceName; }; - std::string_view const PackageName() { return m_packageName; }; - std::vector const &FilterNamespaces() { return m_filterNamespaces; } + bool AllowInternal() const { return m_allowInternal; } + bool IncludeDetail() const { return m_includeDetail; } + bool IncludePrivate() const { return m_includePrivate; } + std::string_view const ReviewName() const { return m_reviewName; }; + std::string_view const ServiceName() const { return m_serviceName; }; + std::string_view const PackageName() const { return m_packageName; }; + std::string_view const SourceRepository() const { return m_repositoryRoot; }; + std::string_view const RootDirectory() const + { + if (m_sourceRoot.empty()) + { + m_sourceRoot = m_currentSourceRoot.string(); + } + return m_sourceRoot; + } + std::vector const& FilterNamespaces() { return m_filterNamespaces; } }; diff --git a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/TextDumper.hpp b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/TextDumper.hpp index 7f5170c38c2..ae1653053c1 100644 --- a/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/TextDumper.hpp +++ b/tools/apiview/parsers/cpp-api-parser/ApiViewProcessor/TextDumper.hpp @@ -16,30 +16,73 @@ class TextDumper : public AstDumper { virtual void InsertWhitespace(int count) override { m_stream << std::string(count, ' '); } virtual void InsertNewline() override { m_stream << std::endl; } - virtual void InsertKeyword(std::string_view const& keyword) override { m_stream << keyword; } - virtual void InsertText(std::string_view const& text) override { m_stream << text; } - virtual void InsertPunctuation(const char punctuation) override { m_stream << punctuation; } - virtual void InsertLineIdMarker() override { m_stream << "// "; } + virtual void InsertKeyword(std::string_view const& keyword) override + { + m_stream << keyword; + UpdateCursor(keyword.size()); + } + virtual void InsertText(std::string_view const& text) override + { + m_stream << text; + UpdateCursor(text.size()); + } + virtual void InsertPunctuation(const char punctuation) override + { + m_stream << punctuation; + UpdateCursor(1); + } + virtual void InsertLineIdMarker() override + { + m_stream << "// "; + UpdateCursor(3); + } virtual void InsertTypeName(std::string_view const& type, std::string_view const&) override { m_stream << type; + UpdateCursor(type.size()); } virtual void InsertMemberName(std::string_view const& member, std::string_view const&) override { m_stream << member; + UpdateCursor(member.size()); + } + virtual void InsertStringLiteral(std::string_view const& str) override + { + m_stream << str; + UpdateCursor(str.size()); + } + virtual void InsertLiteral(std::string_view const& str) override + { + m_stream << str; + UpdateCursor(str.size()); + } + virtual void InsertIdentifier(std::string_view const& str) override + { + m_stream << str; + UpdateCursor(str.size()); + } + virtual void InsertComment(std::string_view const& comment) override + { + m_stream << comment; + UpdateCursor(comment.size()); + } + virtual void AddDocumentRangeStart() override + { + m_stream << "/* ** START DOCUMENTATION RANGE ** */" << std::endl; + } + virtual void AddDocumentRangeEnd() override + { + m_stream << "/* ** END DOCUMENTATION RANGE ** */" << std::endl; + } + virtual void AddExternalLinkStart(std::string_view const& linkValue) override + { + m_stream << "**LINK** ** LINK **"; } - virtual void InsertStringLiteral(std::string_view const& str) override { m_stream << str; } - virtual void InsertLiteral(std::string_view const& str) override { m_stream << str; } - virtual void InsertIdentifier(std::string_view const& str) override { m_stream << str; } - virtual void InsertComment(std::string_view const& comment) override { m_stream << comment; } - virtual void AddDocumentRangeStart() override { m_stream << "/*"; } - virtual void AddDocumentRangeEnd() override { m_stream << "*/"; } + virtual void AddExternalLinkEnd() override { m_stream << "**LINK****LINK**"; } virtual void AddDeprecatedRangeStart() override { m_stream << "/* ** DEPRECATED **"; } virtual void AddDeprecatedRangeEnd() override { m_stream << "/* ** DEPRECATED ** */"; } virtual void AddDiffRangeStart() override { m_stream << "/* ** DIFF **"; } virtual void AddDiffRangeEnd() override { m_stream << " ** DIFF ** */"; } - virtual void AddInheritanceInfoStart() override { m_stream << "/* ** INHERITANCE **"; } - virtual void AddInheritanceInfoEnd() override { m_stream << " ** INHERITANCE ** */"; } void DoDumpHierarchyNode( std::shared_ptr const& node, diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt b/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt index 97a388c658c..05d2a9d071f 100644 --- a/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/CMakeLists.txt @@ -19,7 +19,7 @@ add_executable(parseTests TestCases/ExpressionTests.cpp TestCases/UsingNamespace.cpp TestCases/DestructorTests.cpp -) + TestCases/DocumentationTests.cpp) add_dependencies(parseTests ApiViewProcessor) target_include_directories(parseTests PRIVATE ${ApiViewProcessor_SOURCE_DIR}) diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DocumentationTests.cpp b/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DocumentationTests.cpp new file mode 100644 index 00000000000..aa55e6ece1e --- /dev/null +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/TestCases/DocumentationTests.cpp @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file DoxygenDemo.hpp + * @brief A class skeleton which demonstrates all the doxygen special commands. + * + * This file contains the class skeleton which demonstrates all the doxygen special commands. + * + * @author [Azure SDK Tools] + * + */ + +#ifndef DOXYGEN_DEMO_HPP +#define DOXYGEN_DEMO_HPP + +#include +#include +#include + +/** + * Global integer value. + */ +extern int external; + +// A class which demontrates all the doxygen documentation features. + +/** + * @brief A class that demonstrates all the doxygen special commands. + * + * This class demonstrates all the doxygen special commands. + * + */ +class DoxygenDemo { +public: + /** + * @brief A constructor. + * + * A more elaborate description of the constructor. + * + */ + DoxygenDemo(); + + /** + * @brief A destructor. + * + * A more elaborate description of the destructor. + * + */ + ~DoxygenDemo(); + + /** + * @brief A normal member taking two arguments and returning an integer value. + * + * This function takes two arguments and returns an integer value. + * + * @param a An integer argument. + * @param s A constant character pointer. + * @see DoxygenDemo() + * @see ~DoxygenDemo() + * @see testMeToo() + * @see publicVar() + * @return The test results. + */ + int testMe(int a, const char* s); + + /** + * @brief A normal member taking no arguments and returning nothing. + * + * This function takes no arguments and returns nothing. + * + * @return Nothing. + */ + void testMeToo(); + + /** + * @brief A public variable. + * + * This is a public variable. + * + */ + int publicVar; + + /** + * @brief A normal member taking no arguments and returning nothing. + * + * This function takes no arguments and returns nothing. + * + * @return Nothing. + * @verbatim + * This is a verbatim directive. + * @endverbatim + * + * \cond + * This is a set of documentation that should be excluded. + * \endcond + */ + void verbatimDirective(); + + /** + * \brief A normal member taking no arguments and returning nothing. + * + * This function takes no @a arguments @p and returns @b nothing. + * + * \return Nothing. + * \code{.c} + * int x = 5; + * int y = 10; + * int z = x + y; + * \endcode + */ + void codeDirective(); + + + /** + * @brief A normal member taking no arguments and returning nothing. + * + * This function takes no arguments and returns nothing. It is a multi-line paragraph ending on a \a break. + * + * This is what happens when you have a hypertext link: An + * example link. + * + * @return \a Nothing. + * + * @link https://www.example.com An example link. @endlink + */ + void linkDirective(); + + +private: + /** + * @brief A private variable. + * + * This is a private variable. + * + */ + int privateVar; + + /** + * @brief \b A normal member taking no arguments and returning nothing. + * + * This function takes no arguments and returns nothing. It is a multi-line paragraph ending on a \a break. + * + * This is what happens when you have a hypertext link: An example link. + * + * @return \a Nothing. + */ + void privateFunction(); +}; + +#endif /* DOXYGEN_DEMO_HPP */ diff --git a/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp b/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp index 33cb79967d0..39b78815755 100644 --- a/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp +++ b/tools/apiview/parsers/cpp-api-parser/ParseTests/tests.cpp @@ -263,8 +263,8 @@ struct NsDumper : AstDumper virtual void AddDeprecatedRangeEnd() override {} virtual void AddDiffRangeStart() override {} virtual void AddDiffRangeEnd() override {} - virtual void AddInheritanceInfoStart() override {} - virtual void AddInheritanceInfoEnd() override {} + virtual void AddExternalLinkStart(std::string_view const& url) override {} + virtual void AddExternalLinkEnd() override {} virtual void DumpTypeHierarchyNode( std::shared_ptr const& node) override { @@ -454,7 +454,7 @@ TEST_F(TestParser, Class1) NsDumper dumper; db->DumpClassDatabase(&dumper); - EXPECT_EQ(44ul, dumper.Messages.size()); + EXPECT_EQ(55ul, dumper.Messages.size()); size_t internalTypes = 0; for (const auto& msg : dumper.Messages) @@ -602,6 +602,32 @@ TEST_F(TestParser, TestDtors) EXPECT_EQ(nonVirtualDestructor, 2ul); } +TEST_F(TestParser, TestDocuments) +{ + ApiViewProcessor processor("tests", R"({ + "sourceFilesToProcess": [ + "DocumentationTests.cpp" + ], + "additionalIncludeDirectories": [], + "additionalCompilerSwitches": null, + "allowInternal": false, + "includeDetail": false, + "includePrivate": false, + "filterNamespace": null +} +)"_json); + + + EXPECT_EQ(processor.ProcessApiView(), 0); + + auto& db = processor.GetClassesDatabase(); + EXPECT_TRUE(SyntaxCheckClassDb(db, "DocumentationTests1.cpp")); + + NsDumper dumper; + db->DumpClassDatabase(&dumper); +} + + #if 0 TEST_F(TestParser, AzureCore1) diff --git a/tools/apiview/parsers/cpp-api-parser/README.md b/tools/apiview/parsers/cpp-api-parser/README.md index 3f624481e5c..3f8ba1eddbf 100644 --- a/tools/apiview/parsers/cpp-api-parser/README.md +++ b/tools/apiview/parsers/cpp-api-parser/README.md @@ -80,6 +80,8 @@ An ApiViewSettings.json file contains the following options: - "filterNamespace" - if present and non-null, represents a set of namespace prefixes which are expected in the package. Types which do not match the filter will generate a warning. +- "sourceRootUrl" - if present and non-null represents the root URL for the ApiView directory. + This URL is used to generate source links in the ApiView tool. ## Implementation Details From ee296190fd0a93c9ce3c8c03b5c42206c9f974b4 Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Fri, 20 Oct 2023 13:29:53 -0700 Subject: [PATCH 78/93] [tsp-client] Add ci.yml (#7162) * add ci.yml * Update tools/tsp-client/ci.yml Co-authored-by: Ben Broderick Phillips * update node version * update script to bash * Update tools/tsp-client/ci.yml Co-authored-by: Mike Harder * remove npm and pnpm install in script * Update tools/tsp-client/ci.yml Co-authored-by: Mike Harder * Update tools/tsp-client/ci.yml Co-authored-by: Mike Harder * Update tools/tsp-client/ci.yml Co-authored-by: Mike Harder --------- Co-authored-by: Ben Broderick Phillips Co-authored-by: Mike Harder --- tools/tsp-client/ci.yml | 120 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tools/tsp-client/ci.yml diff --git a/tools/tsp-client/ci.yml b/tools/tsp-client/ci.yml new file mode 100644 index 00000000000..217709d44d3 --- /dev/null +++ b/tools/tsp-client/ci.yml @@ -0,0 +1,120 @@ +# Node.js +# Build a general Node.js project with npm. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/javascript + +trigger: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - tools/tsp-client + +pr: + branches: + include: + - main + - feature/* + - release/* + - hotfix/* + paths: + include: + - tools/tsp-client + +variables: + - template: ../../eng/pipelines/templates/variables/globals.yml + - name: NodeVersion + value: '18.x' + - name: VAR_ARTIFACT_NAME + value: 'drop' + - name: VAR_BUILD_ARTIFACT_STAGING_DIRECTORY + value: $(Build.ArtifactStagingDirectory) + +stages: + - stage: InstallAndBuild + jobs: + - job: Build + strategy: + matrix: + linux: + OSVmImage: 'MMSUbuntu20.04' + Pool: 'azsdk-pool-mms-ubuntu-2004-general' + mac: + OSVmImage: 'macos-12' + Pool: 'Azure Pipelines' + windows: + OSVmImage: 'MMS2022' + Pool: 'azsdk-pool-mms-win-2022-general' + pool: + name: $(Pool) + vmImage: $(OSVmImage) + steps: + - task: NodeTool@0 + inputs: + versionSpec: '$(NodeVersion)' + displayName: 'Install Node.js' + + - bash: | + npm install + displayName: 'npm install' + workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client + + - bash: | + npm pack + displayName: 'npm pack' + workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client + condition: contains(variables['OSVmImage'], 'ubuntu') + + - bash: 'cp azure-tools-tsp-client-*.tgz $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)' + displayName: 'copy to staging dir' + workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client + condition: contains(variables['OSVmImage'], 'ubuntu') + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)' + ArtifactName: '$(VAR_ARTIFACT_NAME)' + publishLocation: 'Container' + condition: contains(variables['OSVmImage'], 'ubuntu') + + - ${{if ne(variables['Build.Reason'], 'PullRequest')}}: + - stage: Release + dependsOn: InstallAndBuild + condition: succeeded() + jobs: + - job: approve + pool: server + steps: + - task: ManualValidation@0 + inputs: + notifyUsers: 'Click to approve if it''s an expected public release.' + - job: release + dependsOn: approve + condition: and(succeeded(), ne(variables['USER_SKIP_PUBLIC_RELEASE'], 'true')) + steps: + - task: NodeTool@0 + inputs: + versionSpec: '$(NodeVersion)' + displayName: 'Install Node.js' + - task: DownloadBuildArtifacts@0 + inputs: + buildType: 'current' + downloadType: 'single' + artifactName: '$(VAR_ARTIFACT_NAME)' + downloadPath: '$(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)' + - bash: | + echo -e "\e[32m[$(date -u)] LOG: publish the package" + echo "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" >> ~/.npmrc + for file in $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)/$(VAR_ARTIFACT_NAME)/*.tgz + do + echo -e "\e[32m[$(date -u)] LOG: File: $file" + npm publish $file --access public || { echo 'publish $file failed' ; exit 1; } + done + rm ~/.npmrc || { echo 'rm ~/.npmrc failed' ; exit 1; } + displayName: Publish + workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client + From a86eb5f738c1cb0c6ba9a691424f35110a195ba8 Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Fri, 20 Oct 2023 14:51:35 -0700 Subject: [PATCH 79/93] [tsp-client] Update README (#7146) * update readme * add more information for commands * update example * move explanation * quotes * fix * update docs * add installation instructions * release prep * update changelog * update changelog * update changelog date * remove private --- tools/tsp-client/CHANGELOG.md | 5 +++ tools/tsp-client/CONTRIBUTING.md | 21 +++++++++++ tools/tsp-client/LICENSE | 21 +++++++++++ tools/tsp-client/README.md | 62 +++++++++++++++++++++----------- tools/tsp-client/package.json | 7 ++-- 5 files changed, 91 insertions(+), 25 deletions(-) create mode 100644 tools/tsp-client/CHANGELOG.md create mode 100644 tools/tsp-client/CONTRIBUTING.md create mode 100644 tools/tsp-client/LICENSE diff --git a/tools/tsp-client/CHANGELOG.md b/tools/tsp-client/CHANGELOG.md new file mode 100644 index 00000000000..9ba1bce2ee9 --- /dev/null +++ b/tools/tsp-client/CHANGELOG.md @@ -0,0 +1,5 @@ +# Release + +## 2023-10-20 - 0.1.0 + +- Initial Release \ No newline at end of file diff --git a/tools/tsp-client/CONTRIBUTING.md b/tools/tsp-client/CONTRIBUTING.md new file mode 100644 index 00000000000..4381e768bbc --- /dev/null +++ b/tools/tsp-client/CONTRIBUTING.md @@ -0,0 +1,21 @@ +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. \ No newline at end of file diff --git a/tools/tsp-client/LICENSE b/tools/tsp-client/LICENSE new file mode 100644 index 00000000000..d1ca00f20a8 --- /dev/null +++ b/tools/tsp-client/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE \ No newline at end of file diff --git a/tools/tsp-client/README.md b/tools/tsp-client/README.md index b694fc03f10..b9963ef2a60 100644 --- a/tools/tsp-client/README.md +++ b/tools/tsp-client/README.md @@ -2,45 +2,65 @@ A simple command line tool for generating TypeSpec clients. -### Usage +### Installation ``` -tsp-client [options] +npm install @azure-tools/typespec-client-generator-cli ``` -## Options - -### (required) - -The only positional parameter. Specifies the directory to pass to the language emitter. - -### --emitter, -e (required) - -Specifies which language emitter to use. Current choices are "csharp", "java", "javascript", "python", "openapi". +### Usage +``` +tsp-client [options] +``` -Aliases are also available, such as cs, js, py, and ts. +## Commands: +Use one of the supported commands to get started generating clients from a TypeSpec project. +This tool will default to using your current working directory to generate clients in and will +use it to look for relevant configuration files. To specify a different directory, use +the `-o` or `--output-dir` option. -### --mainFile, -m +### init +Initialize the SDK project folder from a tspconfig.yaml. When using this command pass in a path to a local or remote tspconfig.yaml, using the `-c` or `--tsp-config` flag. -Used when specifying a URL to a TSP file directly. Not required if using a `tsp-location.yaml` +### update +Sync and generate client libraries from a TypeSpec project. The `update` command will look for a `tsp-location.yaml` file in your current directory to sync a TypeSpec project and generate a client library. -### --debug, -d +### sync +Sync a TypeSpec project with the parameters specified in tsp-location.yaml. -Enables verbose debug logging to the console. +By default the `sync` command will look for a tsp-location.yaml to get the project details and sync them to a temporary directory called `TempTypeSpecFiles`. Alternately, you can pass in the `--local-spec-repo` flag with the path to your local TypeSpec project to pull those files into your temporary directory. -### --no-cleanup +### generate +Generate a client library from a TypeSpec project. The `generate` command should be run after the `sync` command. `generate` relies on the existence of the `TempTypeSpecFiles` directory created by the `sync` command and on an `emitter-package.json` file checked into your repository at the following path: `/eng/emitter-package.json`. The `emitter-package.json` file is used to install project dependencies and get the appropriate emitter package. -Disables automatic cleanup of the temporary directory where the TSP is written and referenced npm modules are installed. +## Options: +``` + -c, --tsp-config The tspconfig.yaml file to use [string] + --commit Commit to be used for project init or update [string] + -d, --debug Enable debug logging [boolean] + --emitter-options The options to pass to the emitter [string] + -h, --help Show help [boolean] + --local-spec-repo Path to local repository with the TypeSpec project [string] + --save-inputs Don't clean up the temp directory after generation [boolean] + --skip-sync-and-generate Skip sync and generate during project init [boolean] + -o, --output-dir Specify an alternate output directory for the + generated files. Default is your local directory. [string] + --repo Repository where the project is defined for init + or update [string] + -v, --version Show version number [boolean] +``` ## Examples -Generating from a TSP file to a particular directory: +Initializing and generating a new client from a `tspconfig.yaml`: + +> NOTE: The `init` command must be run from the root of the repository. ``` -tsp-client -e openapi -m https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/cognitiveservices/OpenAI.Inference/main.tsp ./temp +tsp-client init -c https://github.com/Azure/azure-rest-api-specs/blob/3bae4e510063fbd777b88ea5eee03c41644bc9da/specification/cognitiveservices/ContentSafety/tspconfig.yaml ``` Generating in a directory that contains a `tsp-location.yaml`: ``` -tsp-client sdk/openai/openai +tsp-client update ``` diff --git a/tools/tsp-client/package.json b/tools/tsp-client/package.json index fdc74426feb..3d8d2054298 100644 --- a/tools/tsp-client/package.json +++ b/tools/tsp-client/package.json @@ -1,14 +1,13 @@ { - "name": "@azure-tools/tsp-client", - "version": "0.0.1", - "private": "true", + "name": "@azure-tools/typespec-client-generator-cli", + "version": "0.1.0", "description": "A tool to generate Azure SDKs from TypeSpec", "main": "dist/index.js", "scripts": { "build": "npm run clean && npm run build:tsc", "build:tsc": "tsc", "clean": "rimraf ./dist ./types", - "example": "npx ts-node src/index.ts -e openapi -m https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/cognitiveservices/OpenAI.Inference/main.tsp ./temp", + "example": "npx ts-node src/index.ts update", "test": "mocha" }, "author": "Microsoft Corporation", From 2a69c8b3ce8584483b21ed92e491b92313bea20b Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Fri, 20 Oct 2023 15:35:20 -0700 Subject: [PATCH 80/93] [tsp-client] Fix copy to staging dir (#7174) --- tools/tsp-client/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tsp-client/ci.yml b/tools/tsp-client/ci.yml index 217709d44d3..d3063a9feaf 100644 --- a/tools/tsp-client/ci.yml +++ b/tools/tsp-client/ci.yml @@ -69,7 +69,7 @@ stages: workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client condition: contains(variables['OSVmImage'], 'ubuntu') - - bash: 'cp azure-tools-tsp-client-*.tgz $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)' + - bash: 'cp azure-tools-typespec-client-generator-cli-*.tgz $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)' displayName: 'copy to staging dir' workingDirectory: $(System.DefaultWorkingDirectory)/tools/tsp-client condition: contains(variables['OSVmImage'], 'ubuntu') From df05d431558f8ac2c2e7d507fc90685d0e6b250b Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Fri, 20 Oct 2023 15:38:40 -0700 Subject: [PATCH 81/93] [tsp-client] Improve local spec support (#7126) * update to use simple-git * fix local spec cp * add simple git to package.json * clean up * style --- tools/tsp-client/package.json | 1 + tools/tsp-client/src/git.ts | 64 ++++++++++++----------------------- tools/tsp-client/src/index.ts | 64 ++++++++++++++++------------------- 3 files changed, 51 insertions(+), 78 deletions(-) diff --git a/tools/tsp-client/package.json b/tools/tsp-client/package.json index 3d8d2054298..bdfa71a65a6 100644 --- a/tools/tsp-client/package.json +++ b/tools/tsp-client/package.json @@ -39,6 +39,7 @@ "@azure/core-rest-pipeline": "^1.12.0", "chalk": "^5.3.0", "prompt-sync": "^4.2.0", + "simple-git": "^3.20.0", "yaml": "^2.3.1" }, "peerDependencies": { diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts index f29bf40d7ef..4376dc7184c 100644 --- a/tools/tsp-client/src/git.ts +++ b/tools/tsp-client/src/git.ts @@ -1,29 +1,18 @@ -import { execSync, spawn } from "child_process"; +import { spawn } from "child_process"; +import { simpleGit } from "simple-git"; -export function getRepoRoot(): string { - return execSync('git rev-parse --show-toplevel').toString().trim(); +export async function getRepoRoot(repoPath: string): Promise { + return await simpleGit(repoPath).revparse(["--show-toplevel"]); } - + export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise { - return new Promise((resolve, reject) => { - const git = spawn("git", ["clone", "--no-checkout", "--filter=tree:0", repo, cloneDir], { - cwd: rootUrl, - stdio: "inherit", - }); - git.once("exit", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`git clone failed exited with code ${code}`)); - } - }); - git.once("error", (err) => { - reject(new Error(`git clone failed with error: ${err}`)); - }); - }); - } - - + await simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => { + if (err) { + throw err; + } + }); +} + export async function sparseCheckout(cloneDir: string): Promise { return new Promise((resolve, reject) => { const git = spawn("git", ["sparse-checkout", "init"], { @@ -42,7 +31,7 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string) }); }); } - + export async function addSpecFiles(cloneDir: string, subDir: string): Promise { return new Promise((resolve, reject) => { const git = spawn("git", ["sparse-checkout", "add", subDir], { @@ -61,22 +50,11 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string) }); }); } - - export async function checkoutCommit(cloneDir: string, commit: string): Promise { - return new Promise((resolve, reject) => { - const git = spawn("git", ["checkout", commit], { - cwd: cloneDir, - stdio: "inherit", - }); - git.once("exit", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`git checkout failed exited with code ${code}`)); - } - }); - git.once("error", (err) => { - reject(new Error(`git checkout failed with error: ${err}`)); - }); - }); - } + +export async function checkoutCommit(cloneDir: string, commit: string): Promise { + await simpleGit(cloneDir).checkout(commit, undefined, (err) => { + if (err) { + throw err; + } + }); +} diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index 5e168338841..1084da21230 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -94,7 +94,7 @@ async function sdkInit( async function syncTspFiles(outputDir: string, localSpecRepo?: string) { const tempRoot = await createTempDirectory(outputDir); - const repoRoot = getRepoRoot(); + const repoRoot = await getRepoRoot(outputDir); Logger.debug(`Repo root is ${repoRoot}`); if (repoRoot === undefined) { throw new Error("Could not find repo root"); @@ -108,29 +108,31 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { } const srcDir = path.join(tempRoot, projectName); await mkdir(srcDir, { recursive: true }); - const cloneDir = path.join(repoRoot, "..", "sparse-spec"); - await mkdir(cloneDir, { recursive: true }); - Logger.debug(`Created temporary sparse-checkout directory ${cloneDir}`); - + if (localSpecRepo) { Logger.debug(`Using local spec directory: ${localSpecRepo}`); - function filter(src: string, dest: string): boolean { - if (src.includes("node_modules") || dest.includes("node_modules")) { + function filter(src: string): boolean { + if (src.includes("node_modules")) { + return false; + } + if (src.includes("tsp-output")) { return false; } return true; } - const cpDir = path.join(cloneDir, directory); - await cp(localSpecRepo, cpDir, { recursive: true, filter: filter }); - // TODO: additional directories not yet supported - // const localSpecRepoRoot = await getRepoRoot(localSpecRepo); - // if (localSpecRepoRoot === undefined) { - // throw new Error("Could not find local spec repo root, please make sure the path is correct"); - // } - // for (const dir of additionalDirectories) { - // await cp(path.join(localSpecRepoRoot, dir), cpDir, { recursive: true, filter: filter }); - // } + await cp(localSpecRepo, srcDir, { recursive: true, filter: filter }); + const localSpecRepoRoot = await getRepoRoot(localSpecRepo); + Logger.info(`Local spec repo root is ${localSpecRepoRoot}`) + if (localSpecRepoRoot === undefined) { + throw new Error("Could not find local spec repo root, please make sure the path is correct"); + } + for (const dir of additionalDirectories) { + await cp(path.join(localSpecRepoRoot, dir), srcDir, { recursive: true, filter: filter }); + } } else { + const cloneDir = path.join(repoRoot, "..", "sparse-spec"); + await mkdir(cloneDir, { recursive: true }); + Logger.debug(`Created temporary sparse-checkout directory ${cloneDir}`); Logger.debug(`Cloning repo to ${cloneDir}`); await cloneRepo(tempRoot, cloneDir, `https://github.com/${repo}.git`); await sparseCheckout(cloneDir); @@ -139,28 +141,20 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { for (const dir of additionalDirectories) { await addSpecFiles(cloneDir, dir); } - await checkoutCommit(cloneDir, commit); - } - - await cp(path.join(cloneDir, directory), srcDir, { recursive: true }); - const emitterPath = path.join(repoRoot, "eng", "emitter-package.json"); - await cp(emitterPath, path.join(srcDir, "package.json"), { recursive: true }); - // FIXME: remove conditional once full support for local spec repo is added - if (localSpecRepo) { - Logger.info("local spec repo does not yet support additional directories"); - } else { + await checkoutCommit(cloneDir, commit); + await cp(path.join(cloneDir, directory), srcDir, { recursive: true }); for (const dir of additionalDirectories) { const dirSplit = dir.split("/"); let projectName = dirSplit[dirSplit.length - 1]; - if (projectName === undefined) { - projectName = "src"; - } - const dirName = path.join(tempRoot, projectName); + const dirName = path.join(tempRoot, projectName!); await cp(path.join(cloneDir, dir), dirName, { recursive: true }); } + Logger.debug(`Removing sparse-checkout directory ${cloneDir}`); + await removeDirectory(cloneDir); } - Logger.debug(`Removing sparse-checkout directory ${cloneDir}`); - await removeDirectory(cloneDir); + + const emitterPath = path.join(repoRoot, "eng", "emitter-package.json"); + await cp(emitterPath, path.join(srcDir, "package.json"), { recursive: true }); } @@ -181,7 +175,7 @@ async function generate({ throw new Error("cannot find project name"); } const srcDir = path.join(tempRoot, projectName); - const emitter = await getEmitterFromRepoConfig(path.join(getRepoRoot(), "eng", "emitter-package.json")); + const emitter = await getEmitterFromRepoConfig(path.join(await getRepoRoot(rootUrl), "eng", "emitter-package.json")); if (!emitter) { throw new Error("emitter is undefined"); } @@ -218,7 +212,7 @@ async function main() { switch (options.command) { case "init": - const emitter = await getEmitterFromRepoConfig(path.join(getRepoRoot(), "eng", "emitter-package.json")); + const emitter = await getEmitterFromRepoConfig(path.join(await getRepoRoot(rootUrl), "eng", "emitter-package.json")); if (!emitter) { throw new Error("Couldn't find emitter-package.json in the repo"); } From bf9d1ffad4d9806ade93c71bd5f511451e12d426 Mon Sep 17 00:00:00 2001 From: catalinaperalta Date: Fri, 20 Oct 2023 16:29:42 -0700 Subject: [PATCH 82/93] add prepack to package.json (#7175) --- tools/tsp-client/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tsp-client/package.json b/tools/tsp-client/package.json index bdfa71a65a6..15b5e53df8f 100644 --- a/tools/tsp-client/package.json +++ b/tools/tsp-client/package.json @@ -8,6 +8,7 @@ "build:tsc": "tsc", "clean": "rimraf ./dist ./types", "example": "npx ts-node src/index.ts update", + "prepack": "npm run build", "test": "mocha" }, "author": "Microsoft Corporation", From 97bbe24290c8819254ce868ce1d2f27d2c6c23b1 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 23 Oct 2023 08:10:09 -0700 Subject: [PATCH 83/93] Update tools for new Codeowners utils (#7172) --- .../Services/GitHubService.cs | 32 ++++++++++++------- tools/identity-resolution/ci.yml | 2 ++ .../identity-resolution.csproj | 6 ++-- .../identity-resolution.sln | 25 +++++++++++++++ tools/notification-configuration/ci.yml | 4 +-- .../notification-configuration.sln | 21 ++++++------ .../notification-creator/Contacts.cs | 10 +++--- .../NotificationConfigurator.cs | 2 +- ...e.Sdk.Tools.PipelineOwnersExtractor.csproj | 3 +- .../Processor.cs | 14 ++++---- .../PipelineOwnersExtractor.sln | 12 +++---- tools/pipeline-owners-extractor/ci.yml | 4 +++ 12 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 tools/identity-resolution/identity-resolution.sln diff --git a/tools/identity-resolution/Services/GitHubService.cs b/tools/identity-resolution/Services/GitHubService.cs index fea85c52a3d..c59e527771b 100644 --- a/tools/identity-resolution/Services/GitHubService.cs +++ b/tools/identity-resolution/Services/GitHubService.cs @@ -5,7 +5,8 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using Azure.Sdk.Tools.CodeOwnersParser; +using Azure.Sdk.Tools.CodeownersUtils.Parsing; +using System.IO; namespace Azure.Sdk.Tools.NotificationConfiguration { @@ -14,10 +15,9 @@ namespace Azure.Sdk.Tools.NotificationConfiguration /// public class GitHubService { - private static readonly HttpClient httpClient = new HttpClient(); private static readonly ConcurrentDictionary> - codeownersFileCache = new ConcurrentDictionary>(); + codeownersFileCache = new(); private readonly ILogger logger; @@ -35,7 +35,7 @@ public GitHubService(ILogger logger) /// /// GitHub repository URL /// Contents fo the located CODEOWNERS file - public async Task> GetCodeownersFileEntries(Uri repoUrl) + public List GetCodeownersFileEntries(Uri repoUrl) { List result; if (codeownersFileCache.TryGetValue(repoUrl.ToString(), out result)) @@ -43,7 +43,7 @@ public async Task> GetCodeownersFileEntries(Uri repoUrl) return result; } - result = await GetCodeownersFileImpl(repoUrl); + result = GetCodeownersFileImpl(repoUrl); codeownersFileCache.TryAdd(repoUrl.ToString(), result); return result; } @@ -53,21 +53,31 @@ public async Task> GetCodeownersFileEntries(Uri repoUrl) /// /// /// - private async Task> GetCodeownersFileImpl(Uri repoUrl) + private List GetCodeownersFileImpl(Uri repoUrl) { // Gets the repo path from the URL var relevantPathParts = repoUrl.Segments.Skip(1).Take(2); var repoPath = string.Join("", relevantPathParts); var codeOwnersUrl = $"https://raw.githubusercontent.com/{repoPath}/main/.github/CODEOWNERS"; - var result = await httpClient.GetAsync(codeOwnersUrl); - if (result.IsSuccessStatusCode) + + try + { + this.logger.LogInformation("Parsing CodeownersEntries from CODEOWNERS file URL = {0}", codeOwnersUrl); + return CodeownersParser.ParseCodeownersFile(codeOwnersUrl); + } + // Thrown by FileHelpers if there was the codeOwnersUrl doesn't point to a valid URL or local file. + catch (ArgumentException) + { + this.logger.LogWarning("Unable to retrieve contents from codeOwnersUrl {0}. Please ensure that the file exists.", codeOwnersUrl); + } + // Thrown by FileHelpers if the codeOwnersUrl was good but was unable to be fetched with retries. + // This is the condition where GitHub is having issues. + catch (FileLoadException fle) { - logger.LogInformation("Retrieved CODEOWNERS file URL = {0}", codeOwnersUrl); - return CodeownersFile.GetCodeownersEntries(await result.Content.ReadAsStringAsync()); + this.logger.LogWarning("{0}", fle.Message); } - logger.LogWarning("Could not retrieve CODEOWNERS file URL = {0} ResponseCode = {1}", codeOwnersUrl, result.StatusCode); return null; } diff --git a/tools/identity-resolution/ci.yml b/tools/identity-resolution/ci.yml index 2f2ba1eeb23..ce1c7c6fb9f 100644 --- a/tools/identity-resolution/ci.yml +++ b/tools/identity-resolution/ci.yml @@ -9,6 +9,7 @@ trigger: paths: include: - tools/identity-resolution + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils pr: branches: @@ -20,6 +21,7 @@ pr: paths: include: - tools/identity-resolution + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils extends: template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml diff --git a/tools/identity-resolution/identity-resolution.csproj b/tools/identity-resolution/identity-resolution.csproj index 61e2bd463bb..24e592eb1aa 100644 --- a/tools/identity-resolution/identity-resolution.csproj +++ b/tools/identity-resolution/identity-resolution.csproj @@ -12,9 +12,9 @@ - - - + + + diff --git a/tools/identity-resolution/identity-resolution.sln b/tools/identity-resolution/identity-resolution.sln new file mode 100644 index 00000000000..f1120b00658 --- /dev/null +++ b/tools/identity-resolution/identity-resolution.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34202.233 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "identity-resolution.csproj", "{9BD15934-C50E-4E33-92C7-56FA60BB20F8}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9BD15934-C50E-4E33-92C7-56FA60BB20F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9BD15934-C50E-4E33-92C7-56FA60BB20F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9BD15934-C50E-4E33-92C7-56FA60BB20F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9BD15934-C50E-4E33-92C7-56FA60BB20F8}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {3D8B639A-3BC6-4B18-974F-D413F644F371} + EndGlobalSection +EndGlobal diff --git a/tools/notification-configuration/ci.yml b/tools/notification-configuration/ci.yml index a7055e35ef1..ae36769a7dc 100644 --- a/tools/notification-configuration/ci.yml +++ b/tools/notification-configuration/ci.yml @@ -9,7 +9,7 @@ trigger: paths: include: - tools/notification-configuration - - tools/code-owners-parser/CodeOwnersParser + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils - tools/identity-resolution/identity-resolution pr: @@ -22,7 +22,7 @@ pr: paths: include: - tools/notification-configuration - - tools/code-owners-parser/CodeOwnersParser + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils - tools/identity-resolution/identity-resolution extends: diff --git a/tools/notification-configuration/notification-configuration.sln b/tools/notification-configuration/notification-configuration.sln index 51eddfbb447..2d2023393da 100644 --- a/tools/notification-configuration/notification-configuration.sln +++ b/tools/notification-configuration/notification-configuration.sln @@ -4,13 +4,12 @@ Microsoft Visual Studio Solution File, Format Version 12.00 VisualStudioVersion = 17.5.33103.201 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration", "notification-creator\Azure.Sdk.Tools.NotificationConfiguration.csproj", "{5759063D-A7B3-4D36-ACF4-5595C2789D27}" + ProjectSection(ProjectDependencies) = postProject + {0B5FEC08-8553-4267-B533-47D498A1B910} = {0B5FEC08-8553-4267-B533-47D498A1B910} + EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.NotificationConfiguration.Tests", "notification-creator.Tests\Azure.Sdk.Tools.NotificationConfiguration.Tests.csproj", "{3097CBB4-ED3C-4273-AC67-F5D189CB94BA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "..\code-owners-parser\CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{A9826C8B-85DF-48DB-8A05-40FB04833C42}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "..\code-owners-parser\Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{2146E1FF-04D1-4B19-9767-C011A73CB40D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "..\identity-resolution\identity-resolution.csproj", "{9805B503-5469-412C-9A0C-F09F504F0ED8}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EBC153AF-0244-4DFB-8084-E6C0ACAA5CF3}" @@ -19,6 +18,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeownersUtils", "..\codeowners-utils\Azure.Sdk.Tools.CodeownersUtils\Azure.Sdk.Tools.CodeownersUtils.csproj", "{0B5FEC08-8553-4267-B533-47D498A1B910}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -33,18 +34,14 @@ Global {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Debug|Any CPU.Build.0 = Debug|Any CPU {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.ActiveCfg = Release|Any CPU {3097CBB4-ED3C-4273-AC67-F5D189CB94BA}.Release|Any CPU.Build.0 = Release|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A9826C8B-85DF-48DB-8A05-40FB04833C42}.Release|Any CPU.Build.0 = Release|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2146E1FF-04D1-4B19-9767-C011A73CB40D}.Release|Any CPU.Build.0 = Release|Any CPU {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9805B503-5469-412C-9A0C-F09F504F0ED8}.Debug|Any CPU.Build.0 = Debug|Any CPU {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.ActiveCfg = Release|Any CPU {9805B503-5469-412C-9A0C-F09F504F0ED8}.Release|Any CPU.Build.0 = Release|Any CPU + {0B5FEC08-8553-4267-B533-47D498A1B910}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0B5FEC08-8553-4267-B533-47D498A1B910}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0B5FEC08-8553-4267-B533-47D498A1B910}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0B5FEC08-8553-4267-B533-47D498A1B910}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/notification-configuration/notification-creator/Contacts.cs b/tools/notification-configuration/notification-creator/Contacts.cs index 3f49e89f59c..6e23c334783 100644 --- a/tools/notification-configuration/notification-creator/Contacts.cs +++ b/tools/notification-configuration/notification-creator/Contacts.cs @@ -4,7 +4,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; -using Azure.Sdk.Tools.CodeOwnersParser; +using Azure.Sdk.Tools.CodeownersUtils.Parsing; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.Build.WebApi; @@ -38,7 +38,7 @@ internal Contacts(GitHubService gitHubService, ILogger log) /// /// See the class comment. /// - public async Task> GetFromBuildDefinitionRepoCodeowners(BuildDefinition buildDefinition) + public List GetFromBuildDefinitionRepoCodeowners(BuildDefinition buildDefinition) { if (buildDefinition.Process.Type != BuildDefinitionYamlProcessType) { @@ -60,7 +60,7 @@ public async Task> GetFromBuildDefinitionRepoCodeowners(BuildDefini return null; } - List codeownersEntries = await gitHubService.GetCodeownersFileEntries(repoUrl); + List codeownersEntries = gitHubService.GetCodeownersFileEntries(repoUrl); if (codeownersEntries == null) { this.log.LogInformation("CODEOWNERS file in '{repoUrl}' not found. Skipping sync.", repoUrl); @@ -79,7 +79,7 @@ public async Task> GetFromBuildDefinitionRepoCodeowners(BuildDefini yamlProcess, codeownersEntries, repoUrl.ToString()); - List contacts = matchingCodeownersEntry.Owners; + List contacts = matchingCodeownersEntry.SourceOwners; this.log.LogInformation( "Found matching contacts (owners) in CODEOWNERS. " + @@ -118,7 +118,7 @@ private CodeownersEntry GetMatchingCodeownersEntry( string repoUrl) { CodeownersEntry matchingCodeownersEntry = - CodeownersFile.GetMatchingCodeownersEntry( + CodeownersParser.GetMatchingCodeownersEntry( process.YamlFilename, codeownersEntries); diff --git a/tools/notification-configuration/notification-creator/NotificationConfigurator.cs b/tools/notification-configuration/notification-creator/NotificationConfigurator.cs index 072d7f8e85e..4e0bef8308f 100644 --- a/tools/notification-configuration/notification-creator/NotificationConfigurator.cs +++ b/tools/notification-configuration/notification-creator/NotificationConfigurator.cs @@ -182,7 +182,7 @@ private async Task SyncTeamWithCodeownersFile( using (logger.BeginScope("Team Name = {0}", team.Name)) { List contacts = - await new Contacts(gitHubService, logger).GetFromBuildDefinitionRepoCodeowners(buildDefinition); + new Contacts(gitHubService, logger).GetFromBuildDefinitionRepoCodeowners(buildDefinition); if (contacts == null) { // assert: the reason for why contacts is null has been already logged. diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj index c2fff291567..bd79d27d089 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Azure.Sdk.Tools.PipelineOwnersExtractor.csproj @@ -1,4 +1,4 @@ - + Exe @@ -13,6 +13,7 @@ + diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs index d21baf5efc9..b096740c922 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs @@ -6,7 +6,7 @@ using System.Threading; using System.Threading.Tasks; -using Azure.Sdk.Tools.CodeOwnersParser; +using Azure.Sdk.Tools.CodeownersUtils.Parsing; using Azure.Sdk.Tools.NotificationConfiguration; using Azure.Sdk.Tools.NotificationConfiguration.Helpers; using Azure.Sdk.Tools.NotificationConfiguration.Services; @@ -120,16 +120,16 @@ await File.WriteAllTextAsync( string buildDefPath = process.YamlFilename; logger.LogInformation("Searching CODEOWNERS for patch matching {Path}", buildDefPath); CodeownersEntry codeownersEntry = - CodeownersFile.GetMatchingCodeownersEntry(buildDefPath, codeownersEntries); + CodeownersParser.GetMatchingCodeownersEntry(buildDefPath, codeownersEntries); codeownersEntry.ExcludeNonUserAliases(); logger.LogInformation( "Matching Path = {Path}, Owner Count = {OwnerCount}", buildDefPath, - codeownersEntry.Owners.Count); + codeownersEntry.SourceOwners.Count); // Get set of team members in the CODEOWNERS file - string[] githubOwners = codeownersEntry.Owners.ToArray(); + string[] githubOwners = codeownersEntry.SourceOwners.ToArray(); List microsoftOwners = new List(); @@ -161,10 +161,10 @@ private async Task>> GetCodeownersEntri { IEnumerable Codeowners)>> tasks = repositoryUrls .Select(SanitizeRepositoryUrl) - .Select(async url => ( + .Select(url => Task.FromResult(( RepositoryUrl: url, - Codeowners: await this.gitHubService.GetCodeownersFileEntries(new Uri(url)) - )); + Codeowners: this.gitHubService.GetCodeownersFileEntries(new Uri(url)) + ))); (string RepositoryUrl, List Codeowners)[] taskResults = await Task.WhenAll(tasks); diff --git a/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln index cc454c2528c..3fb5351bbb8 100644 --- a/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln +++ b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln @@ -5,10 +5,10 @@ VisualStudioVersion = 17.2.32602.215 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "identity-resolution", "..\identity-resolution\identity-resolution.csproj", "{54513498-7FA1-4525-915B-405746FBDDF5}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser", "..\code-owners-parser\CodeOwnersParser\Azure.Sdk.Tools.CodeOwnersParser.csproj", "{F0516B1D-037F-4096-9932-ED8F180EBC5D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.PipelineOwnersExtractor", "Azure.Sdk.Tools.PipelineOwnersExtractor\Azure.Sdk.Tools.PipelineOwnersExtractor.csproj", "{4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeownersUtils", "..\codeowners-utils\Azure.Sdk.Tools.CodeownersUtils\Azure.Sdk.Tools.CodeownersUtils.csproj", "{E6FEA30D-55B7-4C04-AC15-1FB7FBED6664}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -19,14 +19,14 @@ Global {54513498-7FA1-4525-915B-405746FBDDF5}.Debug|Any CPU.Build.0 = Debug|Any CPU {54513498-7FA1-4525-915B-405746FBDDF5}.Release|Any CPU.ActiveCfg = Release|Any CPU {54513498-7FA1-4525-915B-405746FBDDF5}.Release|Any CPU.Build.0 = Release|Any CPU - {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F0516B1D-037F-4096-9932-ED8F180EBC5D}.Release|Any CPU.Build.0 = Release|Any CPU {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Debug|Any CPU.Build.0 = Debug|Any CPU {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Release|Any CPU.ActiveCfg = Release|Any CPU {4813A8E4-BC83-4F2D-8C87-F9AAAA11B207}.Release|Any CPU.Build.0 = Release|Any CPU + {E6FEA30D-55B7-4C04-AC15-1FB7FBED6664}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E6FEA30D-55B7-4C04-AC15-1FB7FBED6664}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E6FEA30D-55B7-4C04-AC15-1FB7FBED6664}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E6FEA30D-55B7-4C04-AC15-1FB7FBED6664}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/pipeline-owners-extractor/ci.yml b/tools/pipeline-owners-extractor/ci.yml index dcef7d951b7..519d87cfaa8 100644 --- a/tools/pipeline-owners-extractor/ci.yml +++ b/tools/pipeline-owners-extractor/ci.yml @@ -9,6 +9,8 @@ trigger: paths: include: - tools/pipeline-owners-extractor + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils + - tools/identity-resolution pr: branches: @@ -20,6 +22,8 @@ pr: paths: include: - tools/pipeline-owners-extractor + - tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils + - tools/identity-resolution extends: template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml From 05f2632f36a66c2d268bfc0c3094cf544cc947a1 Mon Sep 17 00:00:00 2001 From: Jesse Squire Date: Mon, 23 Oct 2023 09:58:26 -0700 Subject: [PATCH 84/93] [JimBot] Clarify initial triage flow (#7147) * [JimBot] Clarify initial triage flow Clarifying the intent of adding "Service Attention" during initial triage is to also trigger the logic associated with the Service Attention rule. --- tools/github-event-processor/RULES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/github-event-processor/RULES.md b/tools/github-event-processor/RULES.md index 96d0fe7dd35..842044db4ba 100644 --- a/tools/github-event-processor/RULES.md +++ b/tools/github-event-processor/RULES.md @@ -148,7 +148,7 @@ This is a stand-alone service providing a REST API which requires a service key - Create the following comment "Thank you for your feedback. Tagging and routing to the team member best able to assist." ELSE - - Add "Service Attention" label to the issue + - Add "Service Attention" label to the issue and apply the logic from the "Service Attention" rule - Create the following comment - "Thank you for your feedback. This has been routed to the support team for assistance." ELSE From 3de9a75b61f4eaae72b0c4c4c7d4aaec6183b5da Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 23 Oct 2023 10:02:03 -0700 Subject: [PATCH 85/93] Set IsPackable for CodeownersUtils to false (#7177) --- .../Azure.Sdk.Tools.CodeownersUtils.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj index b732ed8ca9a..78de08e13be 100644 --- a/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj +++ b/tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Azure.Sdk.Tools.CodeownersUtils.csproj @@ -4,6 +4,7 @@ net6.0 disable disable + false From 3a3174ea4b63fd25e2cba5f5524a832cc7aabbe4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 23 Oct 2023 14:45:58 -0400 Subject: [PATCH 86/93] Minor fixes on arm deployment cleanup (#7179) --- eng/scripts/live-test-resource-cleanup.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index af7617a034d..4770aa3b98e 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -337,7 +337,7 @@ function DeleteArmDeployments([object]$ResourceGroup) { if (!$DeleteArmDeployments) { return } - $toDelete = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Where-Object { $_ -and $_.Outputs?.Count } + $toDelete = @(Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Where-Object { $_ -and $_.Outputs?.Count }) if (!$toDelete -or !$toDelete.Count) { return } @@ -369,6 +369,8 @@ function DeleteOrUpdateResourceGroups() { if ($deleteAfter) { if (HasExpiredDeleteAfterTag $deleteAfter) { $toDelete += $rg + } else { + $toClean += $rg } continue } @@ -410,7 +412,9 @@ function DeleteAndPurgeGroups([array]$toDelete) { # Get purgeable resources already in a deleted state. $purgeableResources = @(Get-PurgeableResources) - Write-Host "Total Resource Groups To Delete: $($toDelete.Count)" + if ($toDelete) { + Write-Host "Total Resource Groups To Delete: $($toDelete.Count)" + } foreach ($rg in $toDelete) { $deleteAfter = GetTag $rg "DeleteAfter" From e318a4c7aa4dd368c6d7f15c368087306c7af884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:21:02 -0400 Subject: [PATCH 87/93] Bump Azure.Identity in /tools/stress-cluster/services/Stress.Watcher/src (#7159) Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.5.0 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.5.0...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../services/Stress.Watcher/src/Stress.Watcher.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stress-cluster/services/Stress.Watcher/src/Stress.Watcher.csproj b/tools/stress-cluster/services/Stress.Watcher/src/Stress.Watcher.csproj index 1226dd0098f..14dcdd1ed3d 100644 --- a/tools/stress-cluster/services/Stress.Watcher/src/Stress.Watcher.csproj +++ b/tools/stress-cluster/services/Stress.Watcher/src/Stress.Watcher.csproj @@ -12,7 +12,7 @@ - + From 2547811676df099baec3ac1770410a1353b75637 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 23 Oct 2023 12:23:30 -0700 Subject: [PATCH 88/93] Update tools versions for CodeownersUtils (#7178) --- eng/pipelines/templates/variables/globals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 808bbdb3278..ec0c912ce6e 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -1,5 +1,5 @@ variables: OfficialBuildId: $(Build.BuildNumber) skipComponentGovernanceDetection: true - NotificationsCreatorVersion: '1.0.0-dev.20230223.2' - PipelineOwnersExtractorVersion: '1.0.0-dev.20230211.1' + NotificationsCreatorVersion: '1.0.0-dev.20231023.2' + PipelineOwnersExtractorVersion: '1.0.0-dev.20231023.2' From cd993341aa47befe450db156821d1dd359c3880d Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:57:54 -0700 Subject: [PATCH 89/93] Add Note about `404` due to capitalization (#7132) * add note about 404ing due to crossplat capitalization of path Co-authored-by: Alan Zimmer <48699787+alzimmermsft@users.noreply.github.com> --- tools/assets-automation/asset-sync/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/assets-automation/asset-sync/README.md b/tools/assets-automation/asset-sync/README.md index 083a77cfe30..16ff0cf382f 100644 --- a/tools/assets-automation/asset-sync/README.md +++ b/tools/assets-automation/asset-sync/README.md @@ -45,6 +45,7 @@ - [Sync Operation details - push](#sync-operation-details---push) - [Integrating the sync script w/ language frameworks](#integrating-the-sync-script-w-language-frameworks) - [Test Run](#test-run) + - [A note regarding cross-plat usage](#a-note-regarding-cross-plat-usage) - [Integration Checklist](#integration-checklist) - [Post-Asset-Move space optimizations](#post-asset-move-space-optimizations) - [Test-Proxy creates seeded body content at playback time](#test-proxy-creates-seeded-body-content-at-playback-time) @@ -593,6 +594,21 @@ So to locally repro this experience: 6. `pip install -r dev_requirements.txt` 7. `pytest` +### A note regarding cross-plat usage + +The test-proxy utilizes the `git` of the system running it to retrieve recordings from the assets repository. This means that when loading a recording, the running file system **matters**. When passing a recording path to the test-proxy, ensure that from client side, capitalization is **consistent** across platforms. Let's work through an example. + +A test is recorded on `windows`. It writes to relative recording path `a/path/to/recording.json`. On `windows` and `mac`, if a user attempts to start playback for `a/path/To/recording.json`, this would **succeed** at the attempt to load the recording from disk. This is due to the act that the OS is not case-sensitive. On a **linux** system, attempting to load `a/path/To/recording.json` will **fail**. + +This is extremely easy to overlook when diagnosing `File not found for playback` issues, as capitalization differences can be difficult to see in context. + +If a dev ends up with an asset tag in this situation, the process to resolve it is fairly straightforward. + +1. Delete the recording in local `.assets` directory. +2. `push` the asset, getting a new tag _without_ the problematic tag being present. +3. Run recordings, ensuring that the capitalization of the file is correct. +4. `push`. Tests will pass in CI now. + ## Integration Checklist What needs to be done on each of the repos to utilize this the sync script? @@ -602,7 +618,7 @@ To utilize the _base_ version of the script, the necessary steps are fairly simp - [ ] Add base assets.json - [ ] Update test-proxy `shim`s to call asset-sync scripts to prepare the test directory prior to tests invoking. -Where the difficulty _really_ lies is in the weird situations that folks will into. +Where the difficulty _really_ lies is in the weird situations that folks will run into. To get further assistance beyond what is documented here, there are a couple options. Internal MS users, refer to the [teams channel](https://teams.microsoft.com/l/channel/19%3ab7c3eda7e0864d059721517174502bdb%40thread.skype/Test-Proxy%2520-%2520Questions%252C%2520Help%252C%2520and%2520Discussion?groupId=3e17dcb0-4257-4a30-b843-77f47f1d4121&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47). For external users, file an issue against this repository and tag label `Asset-Sync`. ## Post-Asset-Move space optimizations From 09e9f8876271d89f5b1fb8ee57eee891ccd86892 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:12:54 -0700 Subject: [PATCH 90/93] fix the issue with reporting for java (#7176) --- .../assets-reporting/generate_assets_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/assets-automation/assets-reporting/generate_assets_report.py b/tools/assets-automation/assets-reporting/generate_assets_report.py index 9779adc1eb2..0f7d88ce199 100644 --- a/tools/assets-automation/assets-reporting/generate_assets_report.py +++ b/tools/assets-automation/assets-reporting/generate_assets_report.py @@ -413,7 +413,9 @@ def generate_java_report() -> ScanResult: result.packages_using_proxy.append(os.path.basename(os.path.dirname(pkg))) result.packages_using_external.append(os.path.basename(os.path.dirname(pkg))) + result.packages = sorted(set(result.packages)) print("done.") + return result From 2978bcba8d85b07387db40d165ea65b6a715d802 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 24 Oct 2023 16:26:10 -0400 Subject: [PATCH 91/93] Check az login for target subscription in stress deploy login (#7185) --- .../scripts/stress-testing/stress-test-deployment-lib.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index dde43649a39..60a567c4e8a 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -46,7 +46,7 @@ function RunOrExitOnFailure() function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushImages) { Write-Host "Logging in to subscription, cluster and container registry" - az account show *> $null + az account show -s "$subscription" *> $null if ($LASTEXITCODE) { RunOrExitOnFailure az login --allow-no-subscriptions } From bda226c0045345450446bd44897313e94227678e Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 24 Oct 2023 16:46:40 -0400 Subject: [PATCH 92/93] Clean up arm deployments with secret parameters (#7183) --- eng/scripts/live-test-resource-cleanup.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index 4770aa3b98e..21dc2a99898 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -337,7 +337,8 @@ function DeleteArmDeployments([object]$ResourceGroup) { if (!$DeleteArmDeployments) { return } - $toDelete = @(Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Where-Object { $_ -and $_.Outputs?.Count }) + $toDelete = @(Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName ` + | Where-Object { $_ -and ($_.Outputs?.Count -or $_.Parameters?.ContainsKey('testApplicationSecret')) }) if (!$toDelete -or !$toDelete.Count) { return } From ca491bb2fde54ccb2a7ab93982a95ad946e4dbe6 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 24 Oct 2023 16:47:14 -0400 Subject: [PATCH 93/93] Temporarily disable ACS dogfood cleanup (#7186) --- eng/pipelines/live-test-cleanup.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml index 3616d5cacf2..4b0099757b6 100644 --- a/eng/pipelines/live-test-cleanup.yml +++ b/eng/pipelines/live-test-cleanup.yml @@ -35,9 +35,10 @@ parameters: - DisplayName: Dogfood Translation - Resource Cleanup SubscriptionConfigurations: - $(sub-config-translation-int-test-resources) - - DisplayName: Dogfood ACS - Resource Cleanup - SubscriptionConfigurations: - - $(sub-config-communication-int-test-resources-common) + # TODO: re-enable dogfood cleanup after resource deletion issues are solved, to avoid pipeline timeouts + # - DisplayName: Dogfood ACS - Resource Cleanup + # SubscriptionConfigurations: + # - $(sub-config-communication-int-test-resources-common) - DisplayName: AzureCloud ACS - Resource Cleanup SubscriptionConfigurations: - $(sub-config-azure-cloud-test-resources)