From 7e375ac0e2c78a591b5d221251be321d70ef49f9 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 8 Nov 2023 21:52:34 +0000 Subject: [PATCH 1/8] Run cassettes on main Revert local testing changes Test: make magician-check-vcr-cassettes the only step Also collect passing and skipping tests collectResult no longer returns an error Remove extra return value gitignore magician binary Print logs Actually set logPaths and cassettePaths Rework management of environment variables Remove extra ] Also print all_tests.log Echo home Include HOME in env Echo path Use GOCACHE instead of HOME add -vet=off Run all tests, skip printing logs Run 24 tests and upload logs Add -r and remove logs Also upload cassettes Run tests for one resource in recording Run tests for one resource in replaying Also capture PATH Run recording again Clone hashicorp provider instead of mm Run all tests Run replaying Move check cassettes to push-downstream Remove echo PATH change to GA in doc (#9491) Co-authored-by: Edward Sun Refactor magician structs (#9605) * Refactored github interfaces Fixed bug in overriding breaking changes * gofmt * Removed GetPullRequestLabelIDs Use magician for generate comment Fix formatting of breaking changes Keep diff string empty Add missing newline Add copyright notices Add missing space Revert changes from running generate-comment Run cassettes on main Print logs Rework management of environment variables change to GA in doc (#9491) Co-authored-by: Edward Sun git checkout main gcb-generate-diffs-new.yml --- .ci/gcb-push-downstream.yml | 50 ++++ .ci/magician/cmd/check_cassettes.go | 104 +++++++++ .ci/magician/exec/runner.go | 4 + .ci/magician/vcr/tester.go | 345 ++++++++++++++++++++++++++++ 4 files changed, 503 insertions(+) create mode 100644 .ci/magician/cmd/check_cassettes.go create mode 100644 .ci/magician/vcr/tester.go diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml index e52c8947f68b..d9d99573a5f9 100644 --- a/.ci/gcb-push-downstream.yml +++ b/.ci/gcb-push-downstream.yml @@ -181,6 +181,32 @@ steps: args: - $COMMIT_SHA + - name: 'gcr.io/graphite-docker-images/go-plus' + id: magician-check-vcr-cassettes + waitFor: ["tpgb-push"] + entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' + secretEnv: + - "GITHUB_TOKEN" + - "GOOGLE_BILLING_ACCOUNT" + - "GOOGLE_CUST_ID" + - "GOOGLE_FIRESTORE_PROJECT" + - "GOOGLE_IDENTITY_USER" + - "GOOGLE_MASTER_BILLING_ACCOUNT" + - "GOOGLE_ORG" + - "GOOGLE_ORG_2" + - "GOOGLE_ORG_DOMAIN" + - "GOOGLE_PROJECT" + - "GOOGLE_PROJECT_NUMBER" + - "GOOGLE_SERVICE_ACCOUNT" + - "SA_KEY" + - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION" + env: + - "COMMIT_SHA=$COMMIT_SHA" + - "GOOGLE_REGION=us-central1" + - "GOOGLE_ZONE=us-central1-a" + args: + - "check-cassettes" + # set extremely long 1 day timeout, in order to ensure that any jams / backlogs can be cleared. timeout: 86400s options: @@ -191,5 +217,29 @@ availableSecrets: secretManager: - versionName: projects/673497134629/secrets/github-magician-token/versions/latest env: GITHUB_TOKEN + - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest + env: GOOGLE_BILLING_ACCOUNT + - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest + env: GOOGLE_CUST_ID + - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest + env: GOOGLE_FIRESTORE_PROJECT + - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest + env: GOOGLE_IDENTITY_USER + - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest + env: GOOGLE_MASTER_BILLING_ACCOUNT + - versionName: projects/673497134629/secrets/ci-test-org/versions/latest + env: GOOGLE_ORG + - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest + env: GOOGLE_ORG_2 + - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest + env: GOOGLE_ORG_DOMAIN - versionName: projects/673497134629/secrets/ci-test-project/versions/latest env: GOOGLE_PROJECT + - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest + env: GOOGLE_PROJECT_NUMBER + - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest + env: GOOGLE_SERVICE_ACCOUNT + - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest + env: SA_KEY + - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest + env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go new file mode 100644 index 000000000000..f01d5e938773 --- /dev/null +++ b/.ci/magician/cmd/check_cassettes.go @@ -0,0 +1,104 @@ +package cmd + +import ( + "fmt" + "magician/vcr" + "os" + + "github.com/spf13/cobra" +) + +// TODO(trodge): Move this into magician/github along with repo cloning +const githubUsername = "modular-magician" + +var environmentVariables = [...]string{ + "COMMIT_SHA", + "GITHUB_TOKEN", + "GOCACHE", + "GOPATH", + "GOOGLE_BILLING_ACCOUNT", + "GOOGLE_CUST_ID", + "GOOGLE_FIRESTORE_PROJECT", + "GOOGLE_IDENTITY_USER", + "GOOGLE_MASTER_BILLING_ACCOUNT", + "GOOGLE_ORG", + "GOOGLE_ORG_2", + "GOOGLE_ORG_DOMAIN", + "GOOGLE_PROJECT", + "GOOGLE_PROJECT_NUMBER", + "GOOGLE_REGION", + "GOOGLE_SERVICE_ACCOUNT", + "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", + "GOOGLE_ZONE", + "PATH", + "SA_KEY", +} + +var checkCassettesCmd = &cobra.Command{ + Use: "check-cassettes", + Short: "Run VCR tests on downstream main branch", + Long: `This command runs after downstream changes are merged and runs the most recent + VCR cassettes using the newly built beta provider. + + The following environment variables are expected: +` + listEnvironmentVariables() + ` + + It prints a list of tests that failed in replaying mode along with all test output.`, + Run: func(cmd *cobra.Command, args []string) { + env := make(map[string]string, len(environmentVariables)) + for _, ev := range environmentVariables { + val, ok := os.LookupEnv(ev) + if !ok { + fmt.Printf("Did not provide %s environment variable\n", ev) + os.Exit(1) + } + env[ev] = val + } + + t, err := vcr.NewTester(env) + if err != nil { + fmt.Println("Error creating VCR tester: ", err) + os.Exit(1) + } + execCheckCassettes(t, env["GOPATH"], env["GITHUB_TOKEN"], env["COMMIT_SHA"]) + }, +} + +func listEnvironmentVariables() string { + var result string + for i, ev := range environmentVariables { + result += fmt.Sprintf("\t%2d. %s\n", i+1, ev) + } + return result +} + +func execCheckCassettes(t vcr.Tester, goPath, githubToken, commit string) { + if err := t.FetchCassettes(vcr.Beta); err != nil { + fmt.Println("Error fetching cassettes: ", err) + os.Exit(1) + } + + if err := t.CloneProvider(goPath, githubUsername, githubToken, "downstream-pr-"+commit, vcr.Beta); err != nil { + fmt.Println("Error cloning provider: ", err) + os.Exit(1) + } + + result, err := t.Run(vcr.Replaying, vcr.Beta) + if err != nil { + fmt.Println("Error running VCR: ", err) + os.Exit(1) + } + fmt.Println("Failing tests: ", result.FailedTests) + // TODO(trodge) report these failures to bigquery + fmt.Println("Passing tests: ", result.PassedTests) + fmt.Println("Skipping tests: ", result.SkippedTests) + + if err := t.Cleanup(); err != nil { + fmt.Println("Error cleaning up vcr tester: ", err) + os.Exit(1) + } +} + +func init() { + rootCmd.AddCommand(checkCassettesCmd) +} diff --git a/.ci/magician/exec/runner.go b/.ci/magician/exec/runner.go index 4d85ece99189..96ff79609f06 100644 --- a/.ci/magician/exec/runner.go +++ b/.ci/magician/exec/runner.go @@ -64,6 +64,10 @@ func (ar *Runner) RemoveAll(path string) error { return os.RemoveAll(ar.abs(path)) } +func (ar *Runner) Walk(root string, fn WalkFunc) error { + return filepath.Walk(root, fn) +} + // PushDir changes the directory for the runner to the desired path and saves the previous directory in the stack. func (ar *Runner) PushDir(path string) error { if ar.dirStack == nil { diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go new file mode 100644 index 000000000000..e0f20dd4a6ee --- /dev/null +++ b/.ci/magician/vcr/tester.go @@ -0,0 +1,345 @@ +package vcr + +import ( + "fmt" + "io/fs" + "magician/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +type Tester interface { + CloneProvider(goPath, githubUsername, githubToken, branch string, version Version) error + FetchCassettes(version Version) error + Run(mode Mode, version Version) (*Result, error) + Cleanup() error +} + +type Result struct { + PassedTests []string + SkippedTests []string + FailedTests []string +} + +type Version int + +const ( + GA Version = iota + Beta +) + +const numVersions = 2 + +func (v Version) String() string { + switch v { + case GA: + return "ga" + case Beta: + return "beta" + } + return "unknown" +} + +func (v Version) BucketPath() string { + if v == GA { + return "" + } + return v.String() + "/" +} + +// TODO: move this into magician/github +func (v Version) RepoName() string { + switch v { + case GA: + return "terraform-provider-google" + case Beta: + return "terraform-provider-google-beta" + } + return "unknown" +} + +// TODO: move this into magician/github +func (v Version) URL(githubUsername, githubToken string) string { + return fmt.Sprintf("https://%s:%s@github.com/%s/%s", githubUsername, githubToken, githubUsername, v.RepoName()) +} + +type Mode int + +const ( + Replaying Mode = iota + Recording +) + +const numModes = 2 + +func (m Mode) Lower() string { + switch m { + case Replaying: + return "replaying" + case Recording: + return "recording" + } + return "unknown" +} + +func (m Mode) Upper() string { + return strings.ToUpper(m.Lower()) +} + +type logKey struct { + mode Mode + version Version +} + +type vcrTester struct { + env map[string]string // shared environment variables for running tests + r *exec.Runner // for running commands and manipulating files + version Version // either "ga" or "beta", defaults to "ga" + baseDir string // the directory in which this tester was created + saKeyPath string // where sa_key.json is relative to baseDir + cassettePaths map[Version]string // where cassettes are relative to baseDir by version + logPaths map[logKey]string // where logs are relative to baseDir by version and mode + repoPaths map[Version]string // relative paths of already cloned repos by version +} + +const accTestParalellism = 32 + +const replayingTimeout = "240m" + +var testResultsExpression = regexp.MustCompile(`(?m:^--- (PASS|FAIL|SKIP): TestAcc(\w+))`) + +// Create a new tester in the current working directory and write the service account key file. +func NewTester(env map[string]string) (Tester, error) { + r, err := exec.NewRunner() + if err != nil { + return nil, err + } + saKeyPath := "sa_key.json" + if err := r.WriteFile(saKeyPath, env["SA_KEY"]); err != nil { + return nil, err + } + return &vcrTester{ + env: env, + r: r, + baseDir: r.GetCWD(), + saKeyPath: saKeyPath, + cassettePaths: make(map[Version]string, numVersions), + logPaths: make(map[logKey]string, numVersions*numModes), + repoPaths: make(map[Version]string, numVersions), + }, nil +} + +// Clone the built downstream branch of the provider to a local path under the go path and store it by version in the paths map. +func (vt *vcrTester) CloneProvider(goPath, githubUsername, githubToken, branch string, version Version) error { + if _, ok := vt.repoPaths[version]; ok { + // Skip cloning an already cloned repo. + return nil + } + repoPath := filepath.Join(goPath, "src", "github.com", githubUsername, version.RepoName()) + vt.repoPaths[version] = repoPath + if _, err := vt.r.Run("git", []string{"clone", "--branch", branch, version.URL(githubUsername, githubToken), repoPath}, nil); err != nil { + return err + } + return nil +} + +// Fetch the cassettes for the current version if not already fetched. +// Should be run from the base dir. +func (vt *vcrTester) FetchCassettes(version Version) error { + cassettePath, ok := vt.cassettePaths[version] + if ok { + return nil + } + cassettePath = filepath.Join("cassettes", version.String()) + vt.r.Mkdir(cassettePath) + bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%sfixtures/*", version.BucketPath()) + // Fetch the cassettes. + args := []string{"-m", "-q", "cp", bucketPath, cassettePath} + fmt.Println("Fetching cassettes:\n", "gsutil", strings.Join(args, " ")) + if _, err := vt.r.Run("gsutil", args, nil); err != nil { + return err + } + vt.cassettePaths[version] = cassettePath + return nil +} + +// Run the vcr tests in the given mode and provider version and return the result. +// This will overwrite any existing logs for the given mode and version. +func (vt *vcrTester) Run(mode Mode, version Version) (*Result, error) { + lgky := logKey{mode, version} + logPath, ok := vt.logPaths[lgky] + if !ok { + // We've never run this mode and version. + logPath = filepath.Join("testlogs", mode.Lower(), version.String()) + if err := vt.r.Mkdir(logPath); err != nil { + return nil, err + } + vt.logPaths[lgky] = logPath + } + + repoPath, ok := vt.repoPaths[version] + if !ok { + return nil, fmt.Errorf("no repo cloned for version %s in %v", version, vt.repoPaths) + } + if err := vt.r.PushDir(repoPath); err != nil { + return nil, err + } + testDirs, err := vt.googleTestDirectory() + if err != nil { + return nil, err + } + + cassettePath := filepath.Join("cassettes", version.String()) + if mode == Replaying { + cassettePath, ok = vt.cassettePaths[version] + if !ok { + return nil, fmt.Errorf("cassettes not fetched for version %s", version) + } + } + + args := []string{"test"} + args = append(args, testDirs...) + args = append(args, + "-parallel", + strconv.Itoa(accTestParalellism), + "-v", + "-run=TestAcc", + "-timeout", + replayingTimeout, + `-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc`, + "-vet=off", + ) + env := map[string]string{ + "VCR_PATH": filepath.Join(vt.baseDir, cassettePath), + "VCR_MODE": mode.Upper(), + "ACCTEST_PARALLELISM": strconv.Itoa(accTestParalellism), + "GOOGLE_CREDENTIALS": filepath.Join(vt.baseDir, vt.saKeyPath), + "GOOGLE_APPLICATION_CREDENTIALS": filepath.Join(vt.baseDir, vt.saKeyPath), + "GOOGLE_TEST_DIRECTORY": strings.Join(testDirs, " "), + "TF_LOG": "DEBUG", + "TF_LOG_SDK_FRAMEWORK": "INFO", + "TF_LOG_PATH_MASK": filepath.Join(vt.baseDir, logPath, "%s.log"), + "TF_ACC": "1", + "TF_SCHEMA_PANIC_ON_ERROR": "1", + } + for ev, val := range vt.env { + env[ev] = val + } + var printedEnv string + for ev, val := range env { + if ev == "SA_KEY" || ev == "GITHUB_TOKEN" { + val = "{hidden}" + } + printedEnv += fmt.Sprintf("%s=%s\n", ev, val) + } + fmt.Printf(`Running go: + env: +%v + args: +%s +`, printedEnv, strings.Join(args, " ")) + output, err := vt.r.Run("go", args, env) + if err != nil { + // Use error as output for log. + output = fmt.Sprintf("Error replaying tests:\n%v", err) + } + // Leave repo directory. + if err := vt.r.PopDir(); err != nil { + return nil, err + } + + logFileName := filepath.Join(logPath, "all_tests.log") + // Write output (or error) to test log. + if err := vt.r.WriteFile(logFileName, output); err != nil { + return nil, fmt.Errorf("error writing replaying log: %v, test output: %v", err, output) + } + if err := vt.uploadLogs(logPath, "thomasrodgers-vcr-logs"); err != nil { + return nil, fmt.Errorf("error uploading logs: %v", err) + } + return collectResult(output), nil +} + +// Deletes the service account key and the repos. +func (vt *vcrTester) Cleanup() error { + if err := vt.r.RemoveAll(vt.saKeyPath); err != nil { + return err + } + + for _, path := range vt.repoPaths { + if err := vt.r.RemoveAll(path); err != nil { + return err + } + } + return nil +} + +// Returns a list of all directories to run tests in. +// Must be called after changing into the provider dir. +func (vt *vcrTester) googleTestDirectory() ([]string, error) { + var testDirs []string + if allPackages, err := vt.r.Run("go", []string{"list", "./..."}, nil); err != nil { + return nil, err + } else { + for _, dir := range strings.Split(allPackages, "\n") { + if !strings.Contains(dir, "github.com/hashicorp/terraform-provider-google-beta/scripts") { + testDirs = append(testDirs, dir) + } + } + } + return testDirs, nil +} + +// Print all log file names and contents, except for all_tests.log. +// Must be called after running tests. +func (vt *vcrTester) printLogs(logPath string) { + vt.r.Walk(logPath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return nil + } + if info.Name() == "all_tests.log" { + return nil + } + if info.IsDir() { + return nil + } + fmt.Println("======= ", info.Name(), " =======") + if logContent, err := vt.r.ReadFile(path); err == nil { + fmt.Println(logContent) + } + return nil + }) +} + +func (vt *vcrTester) uploadLogs(logPath, logBucket string) error { + bucketPath := fmt.Sprintf("gs://%s/", logBucket) + args := []string{"-m", "-q", "cp", "-r", logPath, bucketPath} + fmt.Println("Uploading logs:\n", "gsutil", strings.Join(args, " ")) + if _, err := vt.r.Run("gsutil", args, nil); err != nil { + return err + } + args = []string{"-m", "-q", "cp", "-r", "cassettes", bucketPath} + fmt.Println("Uploading cassettes:\n", "gsutil", strings.Join(args, " ")) + if _, err := vt.r.Run("gsutil", args, nil); err != nil { + return err + } + return nil +} + +func collectResult(output string) *Result { + matches := testResultsExpression.FindAllStringSubmatch(output, -1) + results := make(map[string][]string, len(matches)) + for _, submatches := range matches { + if len(submatches) != 3 { + fmt.Printf("Warning: unexpected regex match found in test output: %v", submatches) + continue + } + results[submatches[1]] = append(results[submatches[1]], submatches[2]) + } + return &Result{ + FailedTests: results["FAIL"], + PassedTests: results["PASS"], + SkippedTests: results["SKIP"], + } +} From ba1c6d8635d32ca75563b09dae411bc64b13efb3 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 29 Nov 2023 17:42:59 -0800 Subject: [PATCH 2/8] Move Version into provider package and Repo into source package --- .ci/magician/cmd/check_cassettes.go | 29 ++++-- .ci/magician/vcr/tester.go | 134 ++++++++-------------------- 2 files changed, 59 insertions(+), 104 deletions(-) diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index f01d5e938773..ad338fbeb2c6 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -2,6 +2,9 @@ package cmd import ( "fmt" + "magician/exec" + "magician/provider" + "magician/source" "magician/vcr" "os" @@ -55,12 +58,20 @@ var checkCassettesCmd = &cobra.Command{ env[ev] = val } - t, err := vcr.NewTester(env) + rnr, err := exec.NewRunner() + if err != nil { + fmt.Println("Error creating Runner: ", err) + os.Exit(1) + } + + ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr) + + t, err := vcr.NewTester(env, rnr) if err != nil { fmt.Println("Error creating VCR tester: ", err) os.Exit(1) } - execCheckCassettes(t, env["GOPATH"], env["GITHUB_TOKEN"], env["COMMIT_SHA"]) + execCheckCassettes(env["COMMIT_SHA"], t, ctlr) }, } @@ -72,18 +83,24 @@ func listEnvironmentVariables() string { return result } -func execCheckCassettes(t vcr.Tester, goPath, githubToken, commit string) { - if err := t.FetchCassettes(vcr.Beta); err != nil { +func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) { + if err := t.FetchCassettes(provider.Beta); err != nil { fmt.Println("Error fetching cassettes: ", err) os.Exit(1) } - if err := t.CloneProvider(goPath, githubUsername, githubToken, "downstream-pr-"+commit, vcr.Beta); err != nil { + providerRepo := &source.Repo{ + Name: provider.Beta.RepoName(), + Branch: "downstream-pr-"+commit + } + ctlr.SetPath(providerRepo) + if err := ctlr.Clone(providerRepo); err != nil { fmt.Println("Error cloning provider: ", err) os.Exit(1) } + t.SetRepoPath(provider.Beta, providerRepo.Path) - result, err := t.Run(vcr.Replaying, vcr.Beta) + result, err := t.Run(vcr.Replaying, provider.Beta) if err != nil { fmt.Println("Error running VCR: ", err) os.Exit(1) diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go index e0f20dd4a6ee..3740c61a95ab 100644 --- a/.ci/magician/vcr/tester.go +++ b/.ci/magician/vcr/tester.go @@ -4,6 +4,7 @@ import ( "fmt" "io/fs" "magician/exec" + "magician/provider" "path/filepath" "regexp" "strconv" @@ -11,9 +12,9 @@ import ( ) type Tester interface { - CloneProvider(goPath, githubUsername, githubToken, branch string, version Version) error - FetchCassettes(version Version) error - Run(mode Mode, version Version) (*Result, error) + SetRepoPath(version provider.Version, repoPath string) + FetchCassettes(version provider.Version) error + Run(mode Mode, version provider.Version) (*Result, error) Cleanup() error } @@ -23,48 +24,6 @@ type Result struct { FailedTests []string } -type Version int - -const ( - GA Version = iota - Beta -) - -const numVersions = 2 - -func (v Version) String() string { - switch v { - case GA: - return "ga" - case Beta: - return "beta" - } - return "unknown" -} - -func (v Version) BucketPath() string { - if v == GA { - return "" - } - return v.String() + "/" -} - -// TODO: move this into magician/github -func (v Version) RepoName() string { - switch v { - case GA: - return "terraform-provider-google" - case Beta: - return "terraform-provider-google-beta" - } - return "unknown" -} - -// TODO: move this into magician/github -func (v Version) URL(githubUsername, githubToken string) string { - return fmt.Sprintf("https://%s:%s@github.com/%s/%s", githubUsername, githubToken, githubUsername, v.RepoName()) -} - type Mode int const ( @@ -90,18 +49,17 @@ func (m Mode) Upper() string { type logKey struct { mode Mode - version Version + version provider.Version } type vcrTester struct { - env map[string]string // shared environment variables for running tests - r *exec.Runner // for running commands and manipulating files - version Version // either "ga" or "beta", defaults to "ga" - baseDir string // the directory in which this tester was created - saKeyPath string // where sa_key.json is relative to baseDir - cassettePaths map[Version]string // where cassettes are relative to baseDir by version - logPaths map[logKey]string // where logs are relative to baseDir by version and mode - repoPaths map[Version]string // relative paths of already cloned repos by version + env map[string]string // shared environment variables for running tests + rnr *exec.Runner // for running commands and manipulating files + baseDir string // the directory in which this tester was created + saKeyPath string // where sa_key.json is relative to baseDir + cassettePaths map[provider.Version]string // where cassettes are relative to baseDir by version + logPaths map[logKey]string // where logs are relative to baseDir by version and mode + repoPaths map[provider.Version]string // relative paths of already cloned repos by version } const accTestParalellism = 32 @@ -111,54 +69,40 @@ const replayingTimeout = "240m" var testResultsExpression = regexp.MustCompile(`(?m:^--- (PASS|FAIL|SKIP): TestAcc(\w+))`) // Create a new tester in the current working directory and write the service account key file. -func NewTester(env map[string]string) (Tester, error) { - r, err := exec.NewRunner() - if err != nil { - return nil, err - } +func NewTester(env map[string]string, rnr *exec.Runner) (Tester, error) { saKeyPath := "sa_key.json" - if err := r.WriteFile(saKeyPath, env["SA_KEY"]); err != nil { + if err := rnr.WriteFile(saKeyPath, env["SA_KEY"]); err != nil { return nil, err } return &vcrTester{ env: env, - r: r, - baseDir: r.GetCWD(), + rnr: rnr, + baseDir: rnr.GetCWD(), saKeyPath: saKeyPath, - cassettePaths: make(map[Version]string, numVersions), - logPaths: make(map[logKey]string, numVersions*numModes), - repoPaths: make(map[Version]string, numVersions), + cassettePaths: make(map[provider.Version]string, provider.NumVersions), + logPaths: make(map[logKey]string, provider.NumVersions*numModes), + repoPaths: make(map[provider.Version]string, provider.NumVersions), }, nil } -// Clone the built downstream branch of the provider to a local path under the go path and store it by version in the paths map. -func (vt *vcrTester) CloneProvider(goPath, githubUsername, githubToken, branch string, version Version) error { - if _, ok := vt.repoPaths[version]; ok { - // Skip cloning an already cloned repo. - return nil - } - repoPath := filepath.Join(goPath, "src", "github.com", githubUsername, version.RepoName()) +func (vt *vcrTester) SetRepoPath(version provider.Version, repoPath string) { vt.repoPaths[version] = repoPath - if _, err := vt.r.Run("git", []string{"clone", "--branch", branch, version.URL(githubUsername, githubToken), repoPath}, nil); err != nil { - return err - } - return nil } // Fetch the cassettes for the current version if not already fetched. // Should be run from the base dir. -func (vt *vcrTester) FetchCassettes(version Version) error { +func (vt *vcrTester) FetchCassettes(version provider.Version) error { cassettePath, ok := vt.cassettePaths[version] if ok { return nil } cassettePath = filepath.Join("cassettes", version.String()) - vt.r.Mkdir(cassettePath) + vt.rnr.Mkdir(cassettePath) bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%sfixtures/*", version.BucketPath()) // Fetch the cassettes. args := []string{"-m", "-q", "cp", bucketPath, cassettePath} fmt.Println("Fetching cassettes:\n", "gsutil", strings.Join(args, " ")) - if _, err := vt.r.Run("gsutil", args, nil); err != nil { + if _, err := vt.rnr.Run("gsutil", args, nil); err != nil { return err } vt.cassettePaths[version] = cassettePath @@ -167,13 +111,13 @@ func (vt *vcrTester) FetchCassettes(version Version) error { // Run the vcr tests in the given mode and provider version and return the result. // This will overwrite any existing logs for the given mode and version. -func (vt *vcrTester) Run(mode Mode, version Version) (*Result, error) { +func (vt *vcrTester) Run(mode Mode, version provider.Version) (*Result, error) { lgky := logKey{mode, version} logPath, ok := vt.logPaths[lgky] if !ok { // We've never run this mode and version. logPath = filepath.Join("testlogs", mode.Lower(), version.String()) - if err := vt.r.Mkdir(logPath); err != nil { + if err := vt.rnr.Mkdir(logPath); err != nil { return nil, err } vt.logPaths[lgky] = logPath @@ -183,7 +127,7 @@ func (vt *vcrTester) Run(mode Mode, version Version) (*Result, error) { if !ok { return nil, fmt.Errorf("no repo cloned for version %s in %v", version, vt.repoPaths) } - if err := vt.r.PushDir(repoPath); err != nil { + if err := vt.rnr.PushDir(repoPath); err != nil { return nil, err } testDirs, err := vt.googleTestDirectory() @@ -240,19 +184,19 @@ func (vt *vcrTester) Run(mode Mode, version Version) (*Result, error) { args: %s `, printedEnv, strings.Join(args, " ")) - output, err := vt.r.Run("go", args, env) + output, err := vt.rnr.Run("go", args, env) if err != nil { // Use error as output for log. output = fmt.Sprintf("Error replaying tests:\n%v", err) } // Leave repo directory. - if err := vt.r.PopDir(); err != nil { + if err := vt.rnr.PopDir(); err != nil { return nil, err } logFileName := filepath.Join(logPath, "all_tests.log") // Write output (or error) to test log. - if err := vt.r.WriteFile(logFileName, output); err != nil { + if err := vt.rnr.WriteFile(logFileName, output); err != nil { return nil, fmt.Errorf("error writing replaying log: %v, test output: %v", err, output) } if err := vt.uploadLogs(logPath, "thomasrodgers-vcr-logs"); err != nil { @@ -261,17 +205,11 @@ func (vt *vcrTester) Run(mode Mode, version Version) (*Result, error) { return collectResult(output), nil } -// Deletes the service account key and the repos. +// Deletes the service account key. func (vt *vcrTester) Cleanup() error { - if err := vt.r.RemoveAll(vt.saKeyPath); err != nil { + if err := vt.rnr.RemoveAll(vt.saKeyPath); err != nil { return err } - - for _, path := range vt.repoPaths { - if err := vt.r.RemoveAll(path); err != nil { - return err - } - } return nil } @@ -279,7 +217,7 @@ func (vt *vcrTester) Cleanup() error { // Must be called after changing into the provider dir. func (vt *vcrTester) googleTestDirectory() ([]string, error) { var testDirs []string - if allPackages, err := vt.r.Run("go", []string{"list", "./..."}, nil); err != nil { + if allPackages, err := vt.rnr.Run("go", []string{"list", "./..."}, nil); err != nil { return nil, err } else { for _, dir := range strings.Split(allPackages, "\n") { @@ -294,7 +232,7 @@ func (vt *vcrTester) googleTestDirectory() ([]string, error) { // Print all log file names and contents, except for all_tests.log. // Must be called after running tests. func (vt *vcrTester) printLogs(logPath string) { - vt.r.Walk(logPath, func(path string, info fs.FileInfo, err error) error { + vt.rnr.Walk(logPath, func(path string, info fs.FileInfo, err error) error { if err != nil { return nil } @@ -305,7 +243,7 @@ func (vt *vcrTester) printLogs(logPath string) { return nil } fmt.Println("======= ", info.Name(), " =======") - if logContent, err := vt.r.ReadFile(path); err == nil { + if logContent, err := vt.rnr.ReadFile(path); err == nil { fmt.Println(logContent) } return nil @@ -316,12 +254,12 @@ func (vt *vcrTester) uploadLogs(logPath, logBucket string) error { bucketPath := fmt.Sprintf("gs://%s/", logBucket) args := []string{"-m", "-q", "cp", "-r", logPath, bucketPath} fmt.Println("Uploading logs:\n", "gsutil", strings.Join(args, " ")) - if _, err := vt.r.Run("gsutil", args, nil); err != nil { + if _, err := vt.rnr.Run("gsutil", args, nil); err != nil { return err } args = []string{"-m", "-q", "cp", "-r", "cassettes", bucketPath} fmt.Println("Uploading cassettes:\n", "gsutil", strings.Join(args, " ")) - if _, err := vt.r.Run("gsutil", args, nil); err != nil { + if _, err := vt.rnr.Run("gsutil", args, nil); err != nil { return err } return nil From af26b8ff8380b990028f356d736628c8ebf14cc1 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Fri, 8 Dec 2023 16:19:21 -0800 Subject: [PATCH 3/8] Remove second walk func --- .ci/magician/exec/runner.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/magician/exec/runner.go b/.ci/magician/exec/runner.go index 96ff79609f06..4d85ece99189 100644 --- a/.ci/magician/exec/runner.go +++ b/.ci/magician/exec/runner.go @@ -64,10 +64,6 @@ func (ar *Runner) RemoveAll(path string) error { return os.RemoveAll(ar.abs(path)) } -func (ar *Runner) Walk(root string, fn WalkFunc) error { - return filepath.Walk(root, fn) -} - // PushDir changes the directory for the runner to the desired path and saves the previous directory in the stack. func (ar *Runner) PushDir(path string) error { if ar.dirStack == nil { From 624d6f99b6df2e96ca77a85583704c1499840df2 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Fri, 8 Dec 2023 16:19:47 -0800 Subject: [PATCH 4/8] Add , --- .ci/magician/cmd/check_cassettes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index ad338fbeb2c6..801349789d3b 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -90,8 +90,8 @@ func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) { } providerRepo := &source.Repo{ - Name: provider.Beta.RepoName(), - Branch: "downstream-pr-"+commit + Name: provider.Beta.RepoName(), + Branch: "downstream-pr-" + commit, } ctlr.SetPath(providerRepo) if err := ctlr.Clone(providerRepo); err != nil { From 744996a177271cea99722c2b5f8d0a49a01aa9d4 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Thu, 14 Dec 2023 17:49:43 -0800 Subject: [PATCH 5/8] Delete unused variable --- .ci/magician/cmd/check_cassettes.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index 801349789d3b..b41199d06045 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -11,9 +11,6 @@ import ( "github.com/spf13/cobra" ) -// TODO(trodge): Move this into magician/github along with repo cloning -const githubUsername = "modular-magician" - var environmentVariables = [...]string{ "COMMIT_SHA", "GITHUB_TOKEN", From e1a04fb5dd5d5f71d40347f87afbd59487a01064 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Mon, 18 Dec 2023 10:30:12 -0800 Subject: [PATCH 6/8] Rename bucket --- .ci/magician/vcr/tester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go index 3740c61a95ab..0a7e531237bb 100644 --- a/.ci/magician/vcr/tester.go +++ b/.ci/magician/vcr/tester.go @@ -199,7 +199,7 @@ func (vt *vcrTester) Run(mode Mode, version provider.Version) (*Result, error) { if err := vt.rnr.WriteFile(logFileName, output); err != nil { return nil, fmt.Errorf("error writing replaying log: %v, test output: %v", err, output) } - if err := vt.uploadLogs(logPath, "thomasrodgers-vcr-logs"); err != nil { + if err := vt.uploadLogs(logPath, "vcr-check-cassettes"); err != nil { return nil, fmt.Errorf("error uploading logs: %v", err) } return collectResult(output), nil From df9b1f0818e3b5b4ff4d5e376bd7f3c001d055e6 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Mon, 18 Dec 2023 10:32:33 -0800 Subject: [PATCH 7/8] Include numbers of failed, passed, and skipped tests --- .ci/magician/cmd/check_cassettes.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index b41199d06045..84c72d835df2 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -102,10 +102,10 @@ func execCheckCassettes(commit string, t vcr.Tester, ctlr *source.Controller) { fmt.Println("Error running VCR: ", err) os.Exit(1) } - fmt.Println("Failing tests: ", result.FailedTests) + fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests) // TODO(trodge) report these failures to bigquery - fmt.Println("Passing tests: ", result.PassedTests) - fmt.Println("Skipping tests: ", result.SkippedTests) + fmt.Println(len(result.PassedTests), " passed tests: ", result.PassedTests) + fmt.Println(len(result.SkippedTests), " skipped tests: ", result.SkippedTests) if err := t.Cleanup(); err != nil { fmt.Println("Error cleaning up vcr tester: ", err) From 07f36afab1dd4d52b818c94d53c067827dfc1c30 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 20 Dec 2023 21:07:45 +0000 Subject: [PATCH 8/8] Wait for vcr merge instead of tpgb push --- .ci/gcb-push-downstream.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml index d9d99573a5f9..1f0c856a5b9d 100644 --- a/.ci/gcb-push-downstream.yml +++ b/.ci/gcb-push-downstream.yml @@ -175,6 +175,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh' secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"] + id: vcr-merge waitFor: ["tpg-push"] env: - BASE_BRANCH=$BRANCH_NAME @@ -183,7 +184,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: magician-check-vcr-cassettes - waitFor: ["tpgb-push"] + waitFor: ["vcr-merge"] entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' secretEnv: - "GITHUB_TOKEN"