Magician check cassettes new

.ci/gcb-push-downstream.yml

@@ -181,6 +181,32 @@ steps:
       args:
           - $COMMIT_SHA
 
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: magician-check-vcr-cassettes
+      waitFor: ["tpgb-push"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv:
+        - "GITHUB_TOKEN"
+        - "GOOGLE_BILLING_ACCOUNT"
+        - "GOOGLE_CUST_ID"
+        - "GOOGLE_FIRESTORE_PROJECT"
+        - "GOOGLE_IDENTITY_USER"
+        - "GOOGLE_MASTER_BILLING_ACCOUNT"
+        - "GOOGLE_ORG"
+        - "GOOGLE_ORG_2"
+        - "GOOGLE_ORG_DOMAIN"
+        - "GOOGLE_PROJECT"
+        - "GOOGLE_PROJECT_NUMBER"
+        - "GOOGLE_SERVICE_ACCOUNT"
+        - "SA_KEY"
+        - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
+      env:
+        - "COMMIT_SHA=$COMMIT_SHA"
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+      args:
+        - "check-cassettes"
+
 # set extremely long 1 day timeout, in order to ensure that any jams / backlogs can be cleared.
 timeout: 86400s
 options:
@@ -191,5 +217,29 @@ availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
       env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
+      env: GOOGLE_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest
+      env: GOOGLE_CUST_ID
+    - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest
+      env: GOOGLE_FIRESTORE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest
+      env: GOOGLE_IDENTITY_USER
+    - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest
+      env: GOOGLE_MASTER_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-org/versions/latest
+      env: GOOGLE_ORG
+    - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest
+      env: GOOGLE_ORG_2
+    - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest
+      env: GOOGLE_ORG_DOMAIN
     - versionName: projects/673497134629/secrets/ci-test-project/versions/latest
       env: GOOGLE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest
+      env: GOOGLE_PROJECT_NUMBER
+    - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest
+      env: GOOGLE_SERVICE_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest
+      env: SA_KEY
+    - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
+      env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION

.ci/magician/cmd/check_cassettes.go

@@ -0,0 +1,118 @@
+package cmd
+
+import (
+	"fmt"
+	"magician/exec"
+	"magician/provider"
+	"magician/source"
+	"magician/vcr"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var environmentVariables = [...]string{
+	"COMMIT_SHA",
+	"GITHUB_TOKEN",
+	"GOCACHE",
+	"GOPATH",
+	"GOOGLE_BILLING_ACCOUNT",
+	"GOOGLE_CUST_ID",
+	"GOOGLE_FIRESTORE_PROJECT",
+	"GOOGLE_IDENTITY_USER",
+	"GOOGLE_MASTER_BILLING_ACCOUNT",
+	"GOOGLE_ORG",
+	"GOOGLE_ORG_2",
+	"GOOGLE_ORG_DOMAIN",
+	"GOOGLE_PROJECT",
+	"GOOGLE_PROJECT_NUMBER",
+	"GOOGLE_REGION",
+	"GOOGLE_SERVICE_ACCOUNT",
+	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
+	"GOOGLE_ZONE",
+	"PATH",
+	"SA_KEY",
+}
+
+var checkCassettesCmd = &cobra.Command{
+	Use:   "check-cassettes",
+	Short: "Run VCR tests on downstream main branch",
+	Long: `This command runs after downstream changes are merged and runs the most recent
+	VCR cassettes using the newly built beta provider.
+
+	The following environment variables are expected:
+` + listEnvironmentVariables() + `
+
+	It prints a list of tests that failed in replaying mode along with all test output.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		env := make(map[string]string, len(environmentVariables))
+		for _, ev := range environmentVariables {
+			val, ok := os.LookupEnv(ev)
+			if !ok {
+				fmt.Printf("Did not provide %s environment variable\n", ev)
+				os.Exit(1)
+			}
+			env[ev] = val
+		}
+
+		rnr, err := exec.NewRunner()
+		if err != nil {
+			fmt.Println("Error creating Runner: ", err)
+			os.Exit(1)
+		}
+
+		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr)
+
+		t, err := vcr.NewTester(env, rnr)
+		if err != nil {
+			fmt.Println("Error creating VCR tester: ", err)
+			os.Exit(1)
+		}
+		execCheckCassettes(env["COMMIT_SHA"], t, ctlr)
+	},
+}
+
+func listEnvironmentVariables() string {
+	var result string
+	for i, ev := range environmentVariables {
+		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
+	}
+	return result
+}
+
+func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) {
+	if err := t.FetchCassettes(provider.Beta); err != nil {
+		fmt.Println("Error fetching cassettes: ", err)
+		os.Exit(1)
+	}
+
+	providerRepo := &source.Repo{
+		Name:   provider.Beta.RepoName(),
+		Branch: "downstream-pr-" + commit,
+	}
+	ctlr.SetPath(providerRepo)
+	if err := ctlr.Clone(providerRepo); err != nil {
+		fmt.Println("Error cloning provider: ", err)
+		os.Exit(1)
+	}
+	t.SetRepoPath(provider.Beta, providerRepo.Path)
+
+	result, err := t.Run(vcr.Replaying, provider.Beta)
+	if err != nil {
+		fmt.Println("Error running VCR: ", err)
+		os.Exit(1)
+	}
+	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)
+	// TODO(trodge) report these failures to bigquery
+	fmt.Println(len(result.PassedTests), " passed tests: ", result.PassedTests)
+	fmt.Println(len(result.SkippedTests), " skipped tests: ", result.SkippedTests)
+
+	if err := t.Cleanup(); err != nil {
+		fmt.Println("Error cleaning up vcr tester: ", err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	rootCmd.AddCommand(checkCassettesCmd)
+}