Verify release action (#977)

Add new job in our release workflows to verify that our released SDKs and provider are usable. This will have to be configured into each provider as we will need a program specified in the local repo for each runtime we want to verify. Ideally this will include: nodejs, python, dotnet and Go. Without configuration, the job will not do any verification. There's an example of the available configuration as documentation in the `bridged-provider.config.yaml`. The verify-release workflow is also runnable directly if we want to do a one-off verification. See the [pulumi/verify-provider-release](https://github.com/pulumi/verify-provider-release) action for the underlying implementation. Fixes #972 [Example run without configuration](https://github.com/pulumi/pulumi-xyz/actions/runs/9604446619) skips the job: ![image](https://github.com/pulumi/ci-mgmt/assets/331676/a14dca95-5450-4ccc-b5f2-d2ff3cc9f276) [Example prerelease with nodejs configuration](https://github.com/pulumi/pulumi-xyz/actions/runs/9604468261) runs on Ubuntu and Windows: ![image](https://github.com/pulumi/ci-mgmt/assets/331676/80b11edf-94da-4343-a0f3-163ad40a7d7f)
pulumi · Jun 21, 2024 · 7c1f052 · 7c1f052
1 parent f98a516
commit 7c1f052
Show file tree

Hide file tree

Showing 13 changed files with 436 additions and 12 deletions.
diff --git a/provider-ci/internal/pkg/templates/bridged-provider.config.yaml b/provider-ci/internal/pkg/templates/bridged-provider.config.yaml
@@ -103,3 +103,10 @@ worktreeAllowedChanges: |-
   sdk/go/**/pulumiUtilities.go
   sdk/nodejs/package.json
   sdk/python/pyproject.toml
+
+# Set a path for each language example to enable the test
+# releaseVerification:
+  # nodejs: examples/simple-nodejs
+  # python: examples/simple-python
+  # dotnet: examples/simple-dotnet
+  # go: exampels/simple-go
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml
@@ -325,6 +325,20 @@ jobs:
 #{{- if .Config.extraTests }}#
 #{{ .Config.extraTests | toYaml | indent 2 }}#
 #{{ end }}#
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      # Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
+      enableMacosRunner: false
+
 name: prerelease
 on:
   push:

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
   #{{ if .Config.publishRegistry -}}#
   create_docs_build:
     name: create_docs_build
-    needs: tag_sdk
+    needs: publish_go_sdk
     runs-on: #{{ .Config.runner.default }}#
     steps:
       - name: Dispatch Metadata build
@@ -141,8 +141,8 @@ jobs:
       if: failure()
       name: Send Publish Failure To Slack
       uses: #{{ .Config.actionVersions.slackNotification }}#
-  tag_sdk:
-    name: tag_sdk
+  publish_go_sdk:
+    name: publish_go_sdk
     needs:
       - prerequisites
       - publish_sdk
@@ -367,6 +367,19 @@ jobs:
 #{{- if .Config.extraTests }}#
 #{{ .Config.extraTests | toYaml | indent 2 }}#
 #{{ end }}#
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      enableMacosRunner: true
+
 name: release
 on:
   push:

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/verify-release.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/verify-release.yml
@@ -0,0 +1,95 @@
+name: "Verify Release"
+
+on:
+  workflow_dispatch:
+    inputs:
+      providerVersion:
+        description: "The version of the provider to verify"
+        required: true
+        type: string
+      enableMacRunner:
+        description: "Enable the MacOS runner in addition to Linux and Windows. Defaults to 'false'."
+        required: false
+        type: boolean
+  workflow_call:
+    inputs:
+      providerVersion:
+        description: "The version of the provider to verify"
+        required: true
+        type: string
+      enableMacosRunner:
+        description: "Enable the macos-latest runner in addition to ubuntu-latest and windows-latest. Defaults to 'false'."
+        required: false
+        type: boolean
+        default: false
+
+env:
+#{{ .Config.env | toYaml | indent 2 }}#
+
+jobs:
+  verify-release:
+    name: verify-release
+#{{- if not .Config.releaseVerification }}#
+    # We don't have any release verification configurations, so we never run this workflow.
+    # Configure your .ci-mgmt.yaml files to include the release verification configurations e.g.
+    # releaseVerification:
+    #   nodejs: path/to/nodejs/project
+    #   python: path/to/python/project
+    #   dotnet: path/to/dotnet/project
+    #   go: path/to/go/project
+    if: false
+#{{- end }}#
+    strategy:
+      matrix:
+#{{- if .Config.releaseVerification }}#
+        # We always run on Linux and Windows, and optionally on MacOS. This is because MacOS runners have limited availability.
+        # Expression expands to ["ubuntu-latest","windows-latest"] or ["ubuntu-latest","windows-latest","macos-latest"]
+        # GitHub expressions don't have 'if' statements, so we use a ternary operator to conditionally include the MacOS runner suffix.
+        # See the docs for a similar example to this: https://docs.github.com/en/actions/learn-github-actions/expressions#fromjson
+        runner: ${{ fromJSON(format('["ubuntu-latest","windows-latest"{0}]', github.event.inputs.enableMacRunner == 'true' && ',"macos-latest"' || '')) }}
+#{{- else }}#
+        # We don't have any release verification configurations, so we only run on Linux to print warnings to help users configure the release verification.
+        runner: ["ubuntu-latest"]
+#{{- end }}#
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Setup tools
+        uses: ./.github/actions/setup-tools
+#{{- if .Config.releaseVerification.nodejs }}#
+      - name: Verify nodejs release
+        uses: pulumi/verify-provider-release@v1
+        with:
+          runtime: nodejs
+          directory: #{{ .Config.releaseVerification.nodejs }}#
+          provider: #{{ .Config.provider }}#
+          providerVersion: ${{ inputs.providerVersion }}
+#{{- end }}#
+#{{- if .Config.releaseVerification.python }}#
+      - name: Verify python release
+        uses: pulumi/verify-provider-release@v1
+        with:
+          runtime: python
+          directory: #{{ .Config.releaseVerification.python }}#
+          provider: #{{ .Config.provider }}#
+          providerVersion: ${{ inputs.providerVersion }}
+#{{- end }}#
+#{{- if .Config.releaseVerification.dotnet }}#
+      - name: Verify dotnet release
+        uses: pulumi/verify-provider-release@v1
+        with:
+          runtime: dotnet
+          directory: #{{ .Config.releaseVerification.dotnet }}#
+          provider: #{{ .Config.provider }}#
+          providerVersion: ${{ inputs.providerVersion }}
+#{{- end }}#
+#{{- if .Config.releaseVerification.go }}#
+      - name: Verify go release
+        uses: pulumi/verify-provider-release@v1
+        with:
+          runtime: go
+          directory: #{{ .Config.releaseVerification.go }}#
+          provider: #{{ .Config.provider }}#
+          providerVersion: ${{ inputs.providerVersion }}
+#{{- end }}#
diff --git a/provider-ci/test-workflows/aws/.github/workflows/prerelease.yml b/provider-ci/test-workflows/aws/.github/workflows/prerelease.yml
@@ -317,6 +317,20 @@ jobs:
             uses: codecov/codecov-action@v4
       timeout-minutes: 60
 
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      # Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
+      enableMacosRunner: false
+
 name: prerelease
 on:
   push:

diff --git a/provider-ci/test-workflows/aws/.github/workflows/release.yml b/provider-ci/test-workflows/aws/.github/workflows/release.yml
@@ -39,7 +39,7 @@ jobs:
 
   create_docs_build:
     name: create_docs_build
-    needs: tag_sdk
+    needs: publish_go_sdk
     runs-on: ubuntu-latest
     steps:
       - name: Dispatch Metadata build
@@ -146,8 +146,8 @@ jobs:
       if: failure()
       name: Send Publish Failure To Slack
       uses: rtCamp/action-slack-notify@v2
-  tag_sdk:
-    name: tag_sdk
+  publish_go_sdk:
+    name: publish_go_sdk
     needs:
       - prerequisites
       - publish_sdk
@@ -350,6 +350,19 @@ jobs:
             uses: codecov/codecov-action@v4
       timeout-minutes: 60
 
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      enableMacosRunner: true
+
 name: release
 on:
   push:

diff --git a/provider-ci/test-workflows/aws/.github/workflows/verify-release.yml b/provider-ci/test-workflows/aws/.github/workflows/verify-release.yml
@@ -0,0 +1,67 @@
+name: "Verify Release"
+
+on:
+  workflow_dispatch:
+    inputs:
+      providerVersion:
+        description: "The version of the provider to verify"
+        required: true
+        type: string
+      enableMacRunner:
+        description: "Enable the MacOS runner in addition to Linux and Windows. Defaults to 'false'."
+        required: false
+        type: boolean
+  workflow_call:
+    inputs:
+      providerVersion:
+        description: "The version of the provider to verify"
+        required: true
+        type: string
+      enableMacosRunner:
+        description: "Enable the macos-latest runner in addition to ubuntu-latest and windows-latest. Defaults to 'false'."
+        required: false
+        type: boolean
+        default: false
+
+env:
+  AWS_REGION: us-west-2
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+  NUGET_PUBLISH_KEY: ${{ secrets.NUGET_PUBLISH_KEY }}
+  PUBLISH_REPO_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+  PUBLISH_REPO_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+  PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+  PULUMI_API: https://api.pulumi-staging.io
+  PULUMI_GO_DEP_ROOT: ${{ github.workspace }}/..
+  PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget
+  PULUMI_MISSING_DOCS_ERROR: true
+  PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+  PYPI_USERNAME: __token__
+  SIGNING_KEY: ${{ secrets.JAVA_SIGNING_KEY }}
+  SIGNING_KEY_ID: ${{ secrets.JAVA_SIGNING_KEY_ID }}
+  SIGNING_PASSWORD: ${{ secrets.JAVA_SIGNING_PASSWORD }}
+  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  TF_APPEND_USER_AGENT: pulumi
+
+jobs:
+  verify-release:
+    name: verify-release
+    # We don't have any release verification configurations, so we never run this workflow.
+    # Configure your .ci-mgmt.yaml files to include the release verification configurations e.g.
+    # releaseVerification:
+    #   nodejs: path/to/nodejs/project
+    #   python: path/to/python/project
+    #   dotnet: path/to/dotnet/project
+    #   go: path/to/go/project
+    if: false
+    strategy:
+      matrix:
+        # We don't have any release verification configurations, so we only run on Linux to print warnings to help users configure the release verification.
+        runner: ["ubuntu-latest"]
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Setup tools
+        uses: ./.github/actions/setup-tools
diff --git a/provider-ci/test-workflows/cloudflare/.github/workflows/prerelease.yml b/provider-ci/test-workflows/cloudflare/.github/workflows/prerelease.yml
@@ -272,6 +272,20 @@ jobs:
         - dotnet
         - go
         - java
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      # Prelease is run often but we only have 5 concurrent macos runners, so we only test after the stable release.
+      enableMacosRunner: false
+
 name: prerelease
 on:
   push:

diff --git a/provider-ci/test-workflows/cloudflare/.github/workflows/release.yml b/provider-ci/test-workflows/cloudflare/.github/workflows/release.yml
@@ -39,7 +39,7 @@ jobs:
 
   create_docs_build:
     name: create_docs_build
-    needs: tag_sdk
+    needs: publish_go_sdk
     runs-on: ubuntu-latest
     steps:
       - name: Dispatch Metadata build
@@ -147,8 +147,8 @@ jobs:
       if: failure()
       name: Send Publish Failure To Slack
       uses: rtCamp/action-slack-notify@v2
-  tag_sdk:
-    name: tag_sdk
+  publish_go_sdk:
+    name: publish_go_sdk
     needs:
       - prerequisites
       - publish_sdk
@@ -305,6 +305,19 @@ jobs:
         - dotnet
         - go
         - java
+  verify-release:
+    name: verify-release
+    needs:
+      - prerequisites
+      - publish
+      - publish_sdk
+      - publish_go_sdk
+    uses: ./.github/workflows/verify-release.yml
+    secrets: inherit
+    with:
+      providerVersion: ${{ needs.prerequisites.outputs.version }}
+      enableMacosRunner: true
+
 name: release
 on:
   push: