diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml
index e52c8947f68b..1f0c856a5b9d 100644
--- a/.ci/gcb-push-downstream.yml
+++ b/.ci/gcb-push-downstream.yml
@@ -175,12 +175,39 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"]
+      id: vcr-merge
       waitFor: ["tpg-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
           - $COMMIT_SHA
 
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: magician-check-vcr-cassettes
+      waitFor: ["vcr-merge"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv:
+        - "GITHUB_TOKEN"
+        - "GOOGLE_BILLING_ACCOUNT"
+        - "GOOGLE_CUST_ID"
+        - "GOOGLE_FIRESTORE_PROJECT"
+        - "GOOGLE_IDENTITY_USER"
+        - "GOOGLE_MASTER_BILLING_ACCOUNT"
+        - "GOOGLE_ORG"
+        - "GOOGLE_ORG_2"
+        - "GOOGLE_ORG_DOMAIN"
+        - "GOOGLE_PROJECT"
+        - "GOOGLE_PROJECT_NUMBER"
+        - "GOOGLE_SERVICE_ACCOUNT"
+        - "SA_KEY"
+        - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
+      env:
+        - "COMMIT_SHA=$COMMIT_SHA"
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+      args:
+        - "check-cassettes"
+
 # set extremely long 1 day timeout, in order to ensure that any jams / backlogs can be cleared.
 timeout: 86400s
 options:
@@ -191,5 +218,29 @@ availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
       env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
+      env: GOOGLE_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest
+      env: GOOGLE_CUST_ID
+    - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest
+      env: GOOGLE_FIRESTORE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest
+      env: GOOGLE_IDENTITY_USER
+    - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest
+      env: GOOGLE_MASTER_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-org/versions/latest
+      env: GOOGLE_ORG
+    - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest
+      env: GOOGLE_ORG_2
+    - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest
+      env: GOOGLE_ORG_DOMAIN
     - versionName: projects/673497134629/secrets/ci-test-project/versions/latest
       env: GOOGLE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest
+      env: GOOGLE_PROJECT_NUMBER
+    - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest
+      env: GOOGLE_SERVICE_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest
+      env: SA_KEY
+    - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
+      env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go
new file mode 100644
index 000000000000..84c72d835df2
--- /dev/null
+++ b/.ci/magician/cmd/check_cassettes.go
@@ -0,0 +1,118 @@
+package cmd
+
+import (
+	"fmt"
+	"magician/exec"
+	"magician/provider"
+	"magician/source"
+	"magician/vcr"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var environmentVariables = [...]string{
+	"COMMIT_SHA",
+	"GITHUB_TOKEN",
+	"GOCACHE",
+	"GOPATH",
+	"GOOGLE_BILLING_ACCOUNT",
+	"GOOGLE_CUST_ID",
+	"GOOGLE_FIRESTORE_PROJECT",
+	"GOOGLE_IDENTITY_USER",
+	"GOOGLE_MASTER_BILLING_ACCOUNT",
+	"GOOGLE_ORG",
+	"GOOGLE_ORG_2",
+	"GOOGLE_ORG_DOMAIN",
+	"GOOGLE_PROJECT",
+	"GOOGLE_PROJECT_NUMBER",
+	"GOOGLE_REGION",
+	"GOOGLE_SERVICE_ACCOUNT",
+	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
+	"GOOGLE_ZONE",
+	"PATH",
+	"SA_KEY",
+}
+
+var checkCassettesCmd = &cobra.Command{
+	Use:   "check-cassettes",
+	Short: "Run VCR tests on downstream main branch",
+	Long: `This command runs after downstream changes are merged and runs the most recent
+	VCR cassettes using the newly built beta provider.
+
+	The following environment variables are expected:
+` + listEnvironmentVariables() + `
+
+	It prints a list of tests that failed in replaying mode along with all test output.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		env := make(map[string]string, len(environmentVariables))
+		for _, ev := range environmentVariables {
+			val, ok := os.LookupEnv(ev)
+			if !ok {
+				fmt.Printf("Did not provide %s environment variable\n", ev)
+				os.Exit(1)
+			}
+			env[ev] = val
+		}
+
+		rnr, err := exec.NewRunner()
+		if err != nil {
+			fmt.Println("Error creating Runner: ", err)
+			os.Exit(1)
+		}
+
+		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr)
+
+		t, err := vcr.NewTester(env, rnr)
+		if err != nil {
+			fmt.Println("Error creating VCR tester: ", err)
+			os.Exit(1)
+		}
+		execCheckCassettes(env["COMMIT_SHA"], t, ctlr)
+	},
+}
+
+func listEnvironmentVariables() string {
+	var result string
+	for i, ev := range environmentVariables {
+		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
+	}
+	return result
+}
+
+func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) {
+	if err := t.FetchCassettes(provider.Beta); err != nil {
+		fmt.Println("Error fetching cassettes: ", err)
+		os.Exit(1)
+	}
+
+	providerRepo := &source.Repo{
+		Name:   provider.Beta.RepoName(),
+		Branch: "downstream-pr-" + commit,
+	}
+	ctlr.SetPath(providerRepo)
+	if err := ctlr.Clone(providerRepo); err != nil {
+		fmt.Println("Error cloning provider: ", err)
+		os.Exit(1)
+	}
+	t.SetRepoPath(provider.Beta, providerRepo.Path)
+
+	result, err := t.Run(vcr.Replaying, provider.Beta)
+	if err != nil {
+		fmt.Println("Error running VCR: ", err)
+		os.Exit(1)
+	}
+	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)
+	// TODO(trodge) report these failures to bigquery
+	fmt.Println(len(result.PassedTests), " passed tests: ", result.PassedTests)
+	fmt.Println(len(result.SkippedTests), " skipped tests: ", result.SkippedTests)
+
+	if err := t.Cleanup(); err != nil {
+		fmt.Println("Error cleaning up vcr tester: ", err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	rootCmd.AddCommand(checkCassettesCmd)
+}
diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go
new file mode 100644
index 000000000000..0a7e531237bb
--- /dev/null
+++ b/.ci/magician/vcr/tester.go
@@ -0,0 +1,283 @@
+package vcr
+
+import (
+	"fmt"
+	"io/fs"
+	"magician/exec"
+	"magician/provider"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Tester interface {
+	SetRepoPath(version provider.Version, repoPath string)
+	FetchCassettes(version provider.Version) error
+	Run(mode Mode, version provider.Version) (*Result, error)
+	Cleanup() error
+}
+
+type Result struct {
+	PassedTests  []string
+	SkippedTests []string
+	FailedTests  []string
+}
+
+type Mode int
+
+const (
+	Replaying Mode = iota
+	Recording
+)
+
+const numModes = 2
+
+func (m Mode) Lower() string {
+	switch m {
+	case Replaying:
+		return "replaying"
+	case Recording:
+		return "recording"
+	}
+	return "unknown"
+}
+
+func (m Mode) Upper() string {
+	return strings.ToUpper(m.Lower())
+}
+
+type logKey struct {
+	mode    Mode
+	version provider.Version
+}
+
+type vcrTester struct {
+	env           map[string]string           // shared environment variables for running tests
+	rnr           *exec.Runner                // for running commands and manipulating files
+	baseDir       string                      // the directory in which this tester was created
+	saKeyPath     string                      // where sa_key.json is relative to baseDir
+	cassettePaths map[provider.Version]string // where cassettes are relative to baseDir by version
+	logPaths      map[logKey]string           // where logs are relative to baseDir by version and mode
+	repoPaths     map[provider.Version]string // relative paths of already cloned repos by version
+}
+
+const accTestParalellism = 32
+
+const replayingTimeout = "240m"
+
+var testResultsExpression = regexp.MustCompile(`(?m:^--- (PASS|FAIL|SKIP): TestAcc(\w+))`)
+
+// Create a new tester in the current working directory and write the service account key file.
+func NewTester(env map[string]string, rnr *exec.Runner) (Tester, error) {
+	saKeyPath := "sa_key.json"
+	if err := rnr.WriteFile(saKeyPath, env["SA_KEY"]); err != nil {
+		return nil, err
+	}
+	return &vcrTester{
+		env:           env,
+		rnr:           rnr,
+		baseDir:       rnr.GetCWD(),
+		saKeyPath:     saKeyPath,
+		cassettePaths: make(map[provider.Version]string, provider.NumVersions),
+		logPaths:      make(map[logKey]string, provider.NumVersions*numModes),
+		repoPaths:     make(map[provider.Version]string, provider.NumVersions),
+	}, nil
+}
+
+func (vt *vcrTester) SetRepoPath(version provider.Version, repoPath string) {
+	vt.repoPaths[version] = repoPath
+}
+
+// Fetch the cassettes for the current version if not already fetched.
+// Should be run from the base dir.
+func (vt *vcrTester) FetchCassettes(version provider.Version) error {
+	cassettePath, ok := vt.cassettePaths[version]
+	if ok {
+		return nil
+	}
+	cassettePath = filepath.Join("cassettes", version.String())
+	vt.rnr.Mkdir(cassettePath)
+	bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%sfixtures/*", version.BucketPath())
+	// Fetch the cassettes.
+	args := []string{"-m", "-q", "cp", bucketPath, cassettePath}
+	fmt.Println("Fetching cassettes:\n", "gsutil", strings.Join(args, " "))
+	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
+		return err
+	}
+	vt.cassettePaths[version] = cassettePath
+	return nil
+}
+
+// Run the vcr tests in the given mode and provider version and return the result.
+// This will overwrite any existing logs for the given mode and version.
+func (vt *vcrTester) Run(mode Mode, version provider.Version) (*Result, error) {
+	lgky := logKey{mode, version}
+	logPath, ok := vt.logPaths[lgky]
+	if !ok {
+		// We've never run this mode and version.
+		logPath = filepath.Join("testlogs", mode.Lower(), version.String())
+		if err := vt.rnr.Mkdir(logPath); err != nil {
+			return nil, err
+		}
+		vt.logPaths[lgky] = logPath
+	}
+
+	repoPath, ok := vt.repoPaths[version]
+	if !ok {
+		return nil, fmt.Errorf("no repo cloned for version %s in %v", version, vt.repoPaths)
+	}
+	if err := vt.rnr.PushDir(repoPath); err != nil {
+		return nil, err
+	}
+	testDirs, err := vt.googleTestDirectory()
+	if err != nil {
+		return nil, err
+	}
+
+	cassettePath := filepath.Join("cassettes", version.String())
+	if mode == Replaying {
+		cassettePath, ok = vt.cassettePaths[version]
+		if !ok {
+			return nil, fmt.Errorf("cassettes not fetched for version %s", version)
+		}
+	}
+
+	args := []string{"test"}
+	args = append(args, testDirs...)
+	args = append(args,
+		"-parallel",
+		strconv.Itoa(accTestParalellism),
+		"-v",
+		"-run=TestAcc",
+		"-timeout",
+		replayingTimeout,
+		`-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc`,
+		"-vet=off",
+	)
+	env := map[string]string{
+		"VCR_PATH":                       filepath.Join(vt.baseDir, cassettePath),
+		"VCR_MODE":                       mode.Upper(),
+		"ACCTEST_PARALLELISM":            strconv.Itoa(accTestParalellism),
+		"GOOGLE_CREDENTIALS":             filepath.Join(vt.baseDir, vt.saKeyPath),
+		"GOOGLE_APPLICATION_CREDENTIALS": filepath.Join(vt.baseDir, vt.saKeyPath),
+		"GOOGLE_TEST_DIRECTORY":          strings.Join(testDirs, " "),
+		"TF_LOG":                         "DEBUG",
+		"TF_LOG_SDK_FRAMEWORK":           "INFO",
+		"TF_LOG_PATH_MASK":               filepath.Join(vt.baseDir, logPath, "%s.log"),
+		"TF_ACC":                         "1",
+		"TF_SCHEMA_PANIC_ON_ERROR":       "1",
+	}
+	for ev, val := range vt.env {
+		env[ev] = val
+	}
+	var printedEnv string
+	for ev, val := range env {
+		if ev == "SA_KEY" || ev == "GITHUB_TOKEN" {
+			val = "{hidden}"
+		}
+		printedEnv += fmt.Sprintf("%s=%s\n", ev, val)
+	}
+	fmt.Printf(`Running go:
+	env:
+%v
+	args:
+%s
+`, printedEnv, strings.Join(args, " "))
+	output, err := vt.rnr.Run("go", args, env)
+	if err != nil {
+		// Use error as output for log.
+		output = fmt.Sprintf("Error replaying tests:\n%v", err)
+	}
+	// Leave repo directory.
+	if err := vt.rnr.PopDir(); err != nil {
+		return nil, err
+	}
+
+	logFileName := filepath.Join(logPath, "all_tests.log")
+	// Write output (or error) to test log.
+	if err := vt.rnr.WriteFile(logFileName, output); err != nil {
+		return nil, fmt.Errorf("error writing replaying log: %v, test output: %v", err, output)
+	}
+	if err := vt.uploadLogs(logPath, "vcr-check-cassettes"); err != nil {
+		return nil, fmt.Errorf("error uploading logs: %v", err)
+	}
+	return collectResult(output), nil
+}
+
+// Deletes the service account key.
+func (vt *vcrTester) Cleanup() error {
+	if err := vt.rnr.RemoveAll(vt.saKeyPath); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Returns a list of all directories to run tests in.
+// Must be called after changing into the provider dir.
+func (vt *vcrTester) googleTestDirectory() ([]string, error) {
+	var testDirs []string
+	if allPackages, err := vt.rnr.Run("go", []string{"list", "./..."}, nil); err != nil {
+		return nil, err
+	} else {
+		for _, dir := range strings.Split(allPackages, "\n") {
+			if !strings.Contains(dir, "github.com/hashicorp/terraform-provider-google-beta/scripts") {
+				testDirs = append(testDirs, dir)
+			}
+		}
+	}
+	return testDirs, nil
+}
+
+// Print all log file names and contents, except for all_tests.log.
+// Must be called after running tests.
+func (vt *vcrTester) printLogs(logPath string) {
+	vt.rnr.Walk(logPath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.Name() == "all_tests.log" {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		fmt.Println("======= ", info.Name(), " =======")
+		if logContent, err := vt.rnr.ReadFile(path); err == nil {
+			fmt.Println(logContent)
+		}
+		return nil
+	})
+}
+
+func (vt *vcrTester) uploadLogs(logPath, logBucket string) error {
+	bucketPath := fmt.Sprintf("gs://%s/", logBucket)
+	args := []string{"-m", "-q", "cp", "-r", logPath, bucketPath}
+	fmt.Println("Uploading logs:\n", "gsutil", strings.Join(args, " "))
+	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
+		return err
+	}
+	args = []string{"-m", "-q", "cp", "-r", "cassettes", bucketPath}
+	fmt.Println("Uploading cassettes:\n", "gsutil", strings.Join(args, " "))
+	if _, err := vt.rnr.Run("gsutil", args, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func collectResult(output string) *Result {
+	matches := testResultsExpression.FindAllStringSubmatch(output, -1)
+	results := make(map[string][]string, len(matches))
+	for _, submatches := range matches {
+		if len(submatches) != 3 {
+			fmt.Printf("Warning: unexpected regex match found in test output: %v", submatches)
+			continue
+		}
+		results[submatches[1]] = append(results[submatches[1]], submatches[2])
+	}
+	return &Result{
+		FailedTests:  results["FAIL"],
+		PassedTests:  results["PASS"],
+		SkippedTests: results["SKIP"],
+	}
+}