Magician check cassettes new (GoogleCloudPlatform#9630)

* Run cassettes on main

Revert local testing changes

Test: make magician-check-vcr-cassettes the only step

Also collect passing and skipping tests

collectResult no longer returns an error

Remove extra return value

gitignore magician binary

Print logs

Actually set logPaths and cassettePaths

Rework management of environment variables

Remove extra ]

Also print all_tests.log

Echo home

Include HOME in env

Echo path

Use GOCACHE instead of HOME

add -vet=off

Run all tests, skip printing logs

Run 24 tests and upload logs

Add -r and remove logs

Also upload cassettes

Run tests for one resource in recording

Run tests for one resource in replaying

Also capture PATH

Run recording again

Clone hashicorp provider instead of mm

Run all tests

Run replaying

Move check cassettes to push-downstream

Remove echo PATH

change to GA in doc (GoogleCloudPlatform#9491)

Co-authored-by: Edward Sun <[email protected]>

Refactor magician structs (GoogleCloudPlatform#9605)

* Refactored github interfaces

Fixed bug in overriding breaking changes

* gofmt

* Removed GetPullRequestLabelIDs

Use magician for generate comment

Fix formatting of breaking changes

Keep diff string empty

Add missing newline

Add copyright notices

Add missing space

Revert changes from running generate-comment

Run cassettes on main

Print logs

Rework management of environment variables

change to GA in doc (GoogleCloudPlatform#9491)

Co-authored-by: Edward Sun <[email protected]>

git checkout main gcb-generate-diffs-new.yml

* Move Version into provider package and Repo into source package

* Remove second walk func

* Add ,

* Delete unused variable

* Rename bucket

* Include numbers of failed, passed, and skipped tests

* Wait for vcr merge instead of tpgb push

.ci/gcb-push-downstream.yml

@@ -175,12 +175,39 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"]
+      id: vcr-merge
       waitFor: ["tpg-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
           - $COMMIT_SHA
 
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: magician-check-vcr-cassettes
+      waitFor: ["vcr-merge"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv:
+        - "GITHUB_TOKEN"
+        - "GOOGLE_BILLING_ACCOUNT"
+        - "GOOGLE_CUST_ID"
+        - "GOOGLE_FIRESTORE_PROJECT"
+        - "GOOGLE_IDENTITY_USER"
+        - "GOOGLE_MASTER_BILLING_ACCOUNT"
+        - "GOOGLE_ORG"
+        - "GOOGLE_ORG_2"
+        - "GOOGLE_ORG_DOMAIN"
+        - "GOOGLE_PROJECT"
+        - "GOOGLE_PROJECT_NUMBER"
+        - "GOOGLE_SERVICE_ACCOUNT"
+        - "SA_KEY"
+        - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
+      env:
+        - "COMMIT_SHA=$COMMIT_SHA"
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+      args:
+        - "check-cassettes"
+
 # set extremely long 1 day timeout, in order to ensure that any jams / backlogs can be cleared.
 timeout: 86400s
 options:
@@ -191,5 +218,29 @@ availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
       env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
+      env: GOOGLE_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest
+      env: GOOGLE_CUST_ID
+    - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest
+      env: GOOGLE_FIRESTORE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest
+      env: GOOGLE_IDENTITY_USER
+    - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest
+      env: GOOGLE_MASTER_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-org/versions/latest
+      env: GOOGLE_ORG
+    - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest
+      env: GOOGLE_ORG_2
+    - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest
+      env: GOOGLE_ORG_DOMAIN
     - versionName: projects/673497134629/secrets/ci-test-project/versions/latest
       env: GOOGLE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest
+      env: GOOGLE_PROJECT_NUMBER
+    - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest
+      env: GOOGLE_SERVICE_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest
+      env: SA_KEY
+    - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
+      env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION

.ci/magician/cmd/check_cassettes.go

@@ -0,0 +1,118 @@
+package cmd
+
+import (
+	"fmt"
+	"magician/exec"
+	"magician/provider"
+	"magician/source"
+	"magician/vcr"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var environmentVariables = [...]string{
+	"COMMIT_SHA",
+	"GITHUB_TOKEN",
+	"GOCACHE",
+	"GOPATH",
+	"GOOGLE_BILLING_ACCOUNT",
+	"GOOGLE_CUST_ID",
+	"GOOGLE_FIRESTORE_PROJECT",
+	"GOOGLE_IDENTITY_USER",
+	"GOOGLE_MASTER_BILLING_ACCOUNT",
+	"GOOGLE_ORG",
+	"GOOGLE_ORG_2",
+	"GOOGLE_ORG_DOMAIN",
+	"GOOGLE_PROJECT",
+	"GOOGLE_PROJECT_NUMBER",
+	"GOOGLE_REGION",
+	"GOOGLE_SERVICE_ACCOUNT",
+	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
+	"GOOGLE_ZONE",
+	"PATH",
+	"SA_KEY",
+}
+
+var checkCassettesCmd = &cobra.Command{
+	Use:   "check-cassettes",
+	Short: "Run VCR tests on downstream main branch",
+	Long: `This command runs after downstream changes are merged and runs the most recent
+	VCR cassettes using the newly built beta provider.
+
+	The following environment variables are expected:
+` + listEnvironmentVariables() + `
+
+	It prints a list of tests that failed in replaying mode along with all test output.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		env := make(map[string]string, len(environmentVariables))
+		for _, ev := range environmentVariables {
+			val, ok := os.LookupEnv(ev)
+			if !ok {
+				fmt.Printf("Did not provide %s environment variable\n", ev)
+				os.Exit(1)
+			}
+			env[ev] = val
+		}
+
+		rnr, err := exec.NewRunner()
+		if err != nil {
+			fmt.Println("Error creating Runner: ", err)
+			os.Exit(1)
+		}
+
+		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr)
+
+		t, err := vcr.NewTester(env, rnr)
+		if err != nil {
+			fmt.Println("Error creating VCR tester: ", err)
+			os.Exit(1)
+		}
+		execCheckCassettes(env["COMMIT_SHA"], t, ctlr)
+	},
+}
+
+func listEnvironmentVariables() string {
+	var result string
+	for i, ev := range environmentVariables {
+		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
+	}
+	return result
+}
+
+func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) {
+	if err := t.FetchCassettes(provider.Beta); err != nil {
+		fmt.Println("Error fetching cassettes: ", err)
+		os.Exit(1)
+	}
+
+	providerRepo := &source.Repo{
+		Name:   provider.Beta.RepoName(),
+		Branch: "downstream-pr-" + commit,
+	}
+	ctlr.SetPath(providerRepo)
+	if err := ctlr.Clone(providerRepo); err != nil {
+		fmt.Println("Error cloning provider: ", err)
+		os.Exit(1)
+	}
+	t.SetRepoPath(provider.Beta, providerRepo.Path)
+
+	result, err := t.Run(vcr.Replaying, provider.Beta)
+	if err != nil {
+		fmt.Println("Error running VCR: ", err)
+		os.Exit(1)
+	}
+	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)
+	// TODO(trodge) report these failures to bigquery
+	fmt.Println(len(result.PassedTests), " passed tests: ", result.PassedTests)
+	fmt.Println(len(result.SkippedTests), " skipped tests: ", result.SkippedTests)
+
+	if err := t.Cleanup(); err != nil {
+		fmt.Println("Error cleaning up vcr tester: ", err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	rootCmd.AddCommand(checkCassettesCmd)
+}