[CI] Re-enable API and Generator diff. (#11187)

Use the jenkins script as a base to get the PAI & generator from stable
diff back to the CI. Comment is not perfect, but does provide the
required information. The comment can be improve in a later commit since
this is getting too large.

tools/apidiff/.gitignore

@@ -12,4 +12,5 @@ macos-*.md
 maccat-*.md
 xamarin-*.md
 *.stamp
+*.md.tmp
 bundle-*.zip

tools/compare-commits.sh

@@ -214,6 +214,7 @@ find "$OUTPUT_DIR/build" "$OUTPUT_DIR/build-new" '(' \
 	-name 'compiler' -or \
 	-name 'bgen' -or \
 	-name '*.dll' -or \
+	-name '*.editorconfig' -or \
 	-name '*.pdb' -or \
 	-name '*generated-sources' -or \
 	-name 'generated_sources' -or \

tools/devops/automation/scripts/GitHub.psm1

@@ -414,7 +414,10 @@ function New-GitHubSummaryComment {
         $Artifacts="",
 
         [string]
-        $APIDiff=""
+        $APIDiff="",
+
+        [string]
+        $APIGeneratorDiff=""
     )
 
     $envVars = @{
@@ -502,6 +505,23 @@ function New-GitHubSummaryComment {
     } else {
         Write-Host "API diff urls have not been provided."
     }
+    if (-not [string]::IsNullOrEmpty($APIGeneratorDiff)) {
+        Write-Host "Parsing API diff in path $APIGeneratorDiff"
+        if (-not (Test-Path $APIGeneratorDiff -PathType Leaf)) {
+            $sb.AppendLine("Path $APIGeneratorDiff was not found!")
+        } else {
+            $sb.AppendLine("# API & Generator diff")
+            $sb.AppendLine("")
+            # ugly workaround to get decent new lines
+            foreach ($line in Get-Content -Path $APIGeneratorDiff)
+            {
+                $sb.AppendLine($line)
+            }
+            $sb.AppendLine($apidiffcomments)
+        }
+    } else {
+        Write-Host "API & Generator diff comments have not been provided."
+    }
     if (-not [string]::IsNullOrEmpty($Artifacts)) {
         Write-Host "Parsing artifacts"
         if (-not (Test-Path $Artifacts -PathType Leaf)) {

tools/devops/automation/scripts/bash/compare.sh

@@ -0,0 +1,100 @@
+#!/bin/bash -e
+
+#
+# How to run API comparison locally:
+#
+# 1. To compare a specific commit to the previous one, just run this script (./compare.sh).
+# 2. Replicating what happens for a specific PR is a bit more complicated:
+#    a. Fetch the source code for that PR, using "git fetch --no-tags --progress https://github.com/xamarin/xamarin-macios +refs/pull/<PR>/*:refs/remotes/origin/pr/<PR>/*",
+#       so for example this for PR #4614:
+#
+#            git fetch --no-tags --progress https://github.com/xamarin/xamarin-macios +refs/pull/4614/*:refs/remotes/origin/pr/4614/*
+#
+#    b. Checkout the commit you want to compare (typically the branch you created the PR from if it's your own PR).
+#       i) This isn't 100% accurate, since GitHub will build a merged version
+#          of your branch + the target branch. If you want to test _exactly_
+#          the same code as GitHub, checkout the 'origin/pr/<merge>/merge'
+#          branch the previous git fetch command created.
+#
+#    c. Execute 'PR_ID=4614 ./compare.sh'
+#
+
+API_COMPARISON="$WORKSPACE/apicomparison"
+BUILD_URL="$SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$SYSTEM_TEAMPROJECT/_build/index?buildId=$BUILD_BUILDID&view=ms.vss-test-web.test-result-details"
+URL_PREFIX="$VSDROPSPREFIX/$BUILD_BUILDNUMBER/$BUILD_BUILDID/apigeneratordiff/;/"
+API_URL="$URL_PREFIX/api-diff.html"
+GENERATOR_URL="$URL_PREFIX/generator-diff/index.html"
+
+# env var should have been defined by the CI
+if test -z "$XAM_TOP"; then
+    echo "Variable XAM_TOP is missing."
+    exit 1
+fi
+
+cd $XAM_TOP
+
+MARKDOWN_INDENT="&nbsp;&nbsp;&nbsp;&nbsp;"
+echo "*** Comparing API & creating generator diff... ***"
+export COMPARE_FAILURE_FILE=$TMPDIR/api-diff-compare-failures.txt
+report_error ()
+{
+	printf ":fire: [Failed to compare API and create generator diff](%s/console) :fire:\\n" "$BUILD_URL" >> "$WORKSPACE/api-diff-comments.md"
+	if test -f "$COMPARE_FAILURE_FILE"; then
+		sed "s/^/${MARKDOWN_INDENT//&/\\&}/" "$COMPARE_FAILURE_FILE" >> "$WORKSPACE/api-diff-comments.md"
+	fi
+	printf "${MARKDOWN_INDENT}Search for \`Comparing API & creating generator diff\` in the log to view the complete log.\\n" >> "$WORKSPACE/api-diff-comments.md"
+	touch "$WORKSPACE/failure-stamp"
+	rm -f "$COMPARE_FAILURE_FILE"
+	echo "*** Comparing API & creating generator diff failed ***"
+	MESSAGE="*** Comparing API & creating generator diff failed ***"
+	exit 0
+}
+trap report_error ERR
+
+if test -z "$PR_ID"; then
+	BASE=HEAD
+else
+	BASE="origin/pr/$PR_ID/merge"
+fi
+
+if ! git rev-parse "$BASE" >/dev/null 2>&1; then
+	echo "Can't compare API and create generator diff because the pull request has conflicts that must be resolved first (the branch '$BASE' doesn't exist)."
+	MESSAGE="Can't compare API and create generator diff because the pull request has conflicts that must be resolved first (the branch '$BASE' doesn't exist)."
+	printf ":fire: [Failed to compare API and create generator diff because the pull request has conflicts that must be resolved first](%s/console) :fire:\\n" "$BUILD_URL" >> "$WORKSPACE/api-diff-comments.md"
+	exit 0
+fi
+
+./tools/compare-commits.sh --base="$BASE^1" "--failure-file=$COMPARE_FAILURE_FILE"
+
+mkdir -p "$API_COMPARISON"
+
+
+cp -R ./tools/comparison/apidiff/diff "$API_COMPARISON"
+cp    ./tools/comparison/apidiff/*.html "$API_COMPARISON"
+cp -R ./tools/comparison/generator-diff "$API_COMPARISON"
+
+if ! grep "href=" "$API_COMPARISON/api-diff.html" >/dev/null 2>&1; then
+	printf ":white_check_mark: [API Diff (from PR only)](%s) (no change)" "$API_URL" >> "$WORKSPACE/api-diff-comments.md"
+elif perl -0777 -pe 's/<script type="text\/javascript">.*?<.script>/script removed/gs' "$API_COMPARISON/*.html" | grep data-is-breaking; then
+	printf ":warning: [API Diff (from PR only)](%s) (:fire: breaking changes :fire:)" "$API_URL" >> "$WORKSPACE/api-diff-comments.md"
+else
+	printf ":information_source: [API Diff (from PR only)](%s) (please review changes)" "$API_URL" >> "$WORKSPACE/api-diff-comments.md"
+fi
+printf "\\n" >> "$WORKSPACE/api-diff-comments.md"
+
+if ! test -s $API_COMPARISON/generator-diff/generator.diff; then
+	printf ":white_check_mark: [Generator Diff](%s) (no change)" "$GENERATOR_URL" >> "$WORKSPACE/api-diff-comments.md"
+elif grep "^[+-][^+-]" $API_COMPARISON/generator-diff/generator.diff | grep -v "^.[[]assembly: AssemblyInformationalVersion" | grep -v "^[+-][[:space:]]*internal const string Revision =" >/dev/null 2>&1; then
+	printf ":information_source: [Generator Diff](%s) (please review changes)" "$GENERATOR_URL" >> "$WORKSPACE/api-diff-comments.md"
+else
+	printf ":white_check_mark: [Generator Diff](%s) (only version changes)" "$GENERATOR_URL" >> "$WORKSPACE/api-diff-comments.md"
+fi
+printf "\\n" >> "$WORKSPACE/api-diff-comments.md"
+
+cp "$WORKSPACE/api-diff-comments.md" "$API_COMPARISON"
+if test -f "$COMPARE_FAILURE_FILE"; then
+	cp "$COMPARE_FAILURE_FILE" "$API_COMPARISON"
+fi
+MESSAGE="*** Comparing API & creating generator diff completed ***"
+echo "##vso[task.setvariable variable=API_GENERATOR_DIFF_MESSAGE;isOutput=true]$MESSAGE"
+echo "##vso[task.setvariable variable=API_GENERATOR_BUILT;isOutput=true]True"

tools/devops/automation/scripts/bash/local-compare.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+#
+# How to run API comparison locally:
+#
+# 1. To compare a specific commit to the previous one, just run this script (./compare.sh).
+# 2. Replicating what happens for a specific PR is a bit more complicated:
+#    a. Fetch the source code for that PR, using "git fetch --no-tags --progress https://github.com/xamarin/xamarin-macios +refs/pull/<PR>/*:refs/remotes/origin/pr/<PR>/*",
+#       so for example this for PR #4614:
+#
+#            git fetch --no-tags --progress https://github.com/xamarin/xamarin-macios +refs/pull/4614/*:refs/remotes/origin/pr/4614/*
+#
+#    b. Checkout the commit you want to compare (typically the branch you created the PR from if it's your own PR).
+#       i) This isn't 100% accurate, since GitHub will build a merged version
+#          of your branch + the target branch. If you want to test _exactly_
+#          the same code as GitHub, checkout the 'origin/pr/<merge>/merge'
+#          branch the previous git fetch command created.
+#
+#    c. Execute 'ghprbPullId=4614 ./compare.sh'
+#
+
+# set needed env vars, call the actual script
+cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.."
+export WORKSPACE=$(pwd)
+export XAM_TOP="$WORKSPACE"
+
+./tools/devops/automation/scripts/bash/compare.sh

tools/devops/automation/scripts/bash/vsts-compare.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+# set the env vars that are required for the compare to work. XAM_TOP is not set because
+# the yaml takes care of that.
+export WORKSPACE="$BUILD_ARTIFACTSTAGINGDIRECTORY"
+echo "GH PR: $PR_ID"
+cd $XAM_TOP
+
+if [[ $PR_ID ]]; then
+    git fetch --no-tags --progress https://github.com/xamarin/xamarin-macios +refs/pull/$PR_ID/*:refs/remotes/origin/pr/$PR_ID/*
+fi
+
+
+./tools/devops/automation/scripts/bash/compare.sh

tools/devops/automation/templates/build/api-diff.yml

@@ -1,6 +1,39 @@
 # Contains all the different steps to generate the diff API diffs
 
+parameters:
+
+- name: prID
+  type: string 
+  default: '' # default empty, meaning we are building in CI 
+
 steps:
+- bash:  $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/vsts-compare.sh
+  displayName: 'API & Generator comparison'
+  condition: and(succeeded(), contains(variables['configuration.SkipPublicJenkins'], 'False'))
+  name: compare
+  env:
+    BUILD_REVISION: 'jenkins'
+    PR_ID:  ${{ parameters.prID }} # reusing jenkins vars, to be fixed
+
+
+- task: ArchiveFiles@1
+  displayName: 'Archive API & Generator comparison'
+  inputs:
+    rootFolder: $(Build.ArtifactStagingDirectory)/apicomparison
+    includeRootFolder: false
+    archiveFile: '$(Build.ArtifactStagingDirectory)/apicomparison.zip'
+  condition: and(succeeded(), contains(variables['configuration.SkipPublicJenkins'], 'False'))
+  continueOnError: true
+
+- task: PublishPipelineArtifact@1
+  displayName: 'Publish API & Generator comparison'
+  inputs:
+    targetPath: '$(Build.ArtifactStagingDirectory)/apicomparison.zip'
+    artifactName: apicomparison
+  condition: and(succeeded(), contains(variables['configuration.SkipPublicJenkins'], 'False'))
+  continueOnError: true
+
+# order matters, first compare then diff, else git complains
 - bash: |
     report_error ()
     {
@@ -24,6 +57,7 @@ steps:
   continueOnError: true
   env:
     BUILD_REVISION: 'jenkins'
+    PR_ID:  ${{ parameters.prID}} # reusing jenkins vars, to be fixed
 
 - task: ArchiveFiles@1
   displayName: 'Archive API diff (from stable)'

tools/devops/automation/templates/build/build.yml

@@ -294,6 +294,11 @@ steps:
   displayName: 'Build'
   timeoutInMinutes: 180
 
+ # run ASAP so that we do not have any files that got generated confusing git 
+- template: api-diff.yml
+  parameters:
+    prID: variables['PrID']
+
 # build not signed .pkgs for the SDK
 - bash: |
     set -x
@@ -369,8 +374,6 @@ steps:
   condition: and(succeeded(), contains(variables['configuration.RunMacTests'], 'True'))
   continueOnError: true
 
-- template: api-diff.yml
-
 - bash: |
     set -x
     set -e

tools/devops/automation/templates/build/download-artifacts.yml

@@ -28,8 +28,22 @@ steps:
     archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Reports/apidiff-stable/apidiff-stable.zip'
     destinationFolder: '$(System.DefaultWorkingDirectory)/apidiff-stable'
 
+- task: DownloadPipelineArtifact@2
+  displayName: 'Download API & Generator comparison'
+  inputs:
+    patterns: 'apicomparison/apicomparison.zip'
+    allowFailedBuilds: true
+    path: $(System.DefaultWorkingDirectory)/Reports
+
+- task: ExtractFiles@1
+  displayName: 'Extract API & Generator comparison'
+  inputs:
+    archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Reports/apicomparison/apicomparison.zip'
+    destinationFolder: '$(System.DefaultWorkingDirectory)/apicomparison'
+
 - powershell: |
     Write-Host "##vso[task.setvariable variable=STABLE_APIDDIFF_PATH]$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\apidiff-stable"
+    Write-Host "##vso[task.setvariable variable=STABLE_APID_GENERATOR_DIFF_PATH]$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\apicomparison"
   displayName: Publish apidiff paths
   name: apidiff # not to be confused with the displayName, this is used to later use the name of the step to access the output variables from an other job