From 0ddb0d110c2c297b9cf4f154dc49ea1e2e21642c Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Fri, 20 Nov 2020 12:19:14 -0500 Subject: [PATCH 1/4] dev: introduce a general purpose dev-tool for crdb engineers Our Makefile UX has grown organically over the last six years, and is now home to various scripts that are orthogonal to simply building the crdb binary. These include the ability to run linters, building various useful binaries (such as roachprod, roachtest), running different kinds of tests (unit tests, logic tests, acceptance tests), and much more. Now that we're exploring a move towards Bazel for our build system (#55687), we have a chance to tuck away Bazel specific under a general purpose dev-tool. The intent here is to house all day-to-day dev operations under this tool, written in Go. This will be the component that actually replaces our Makefile in its entirety. It'll be (predictably) powered by bazel underneath, but with much nicer UX. It should capturing all common usage patterns in high-level docs. This diff only introduces the scaffolding for this CLI. Release note: None --- Makefile | 1 + pkg/cmd/dev/BUILD.bazel | 25 +++++++++++++++++ pkg/cmd/dev/bench.go | 32 ++++++++++++++++++++++ pkg/cmd/dev/build.go | 34 ++++++++++++++++++++++++ pkg/cmd/dev/generate.go | 36 +++++++++++++++++++++++++ pkg/cmd/dev/lint.go | 32 ++++++++++++++++++++++ pkg/cmd/dev/main.go | 59 +++++++++++++++++++++++++++++++++++++++++ pkg/cmd/dev/test.go | 35 ++++++++++++++++++++++++ 8 files changed, 254 insertions(+) create mode 100644 pkg/cmd/dev/BUILD.bazel create mode 100644 pkg/cmd/dev/bench.go create mode 100644 pkg/cmd/dev/build.go create mode 100644 pkg/cmd/dev/generate.go create mode 100644 pkg/cmd/dev/lint.go create mode 100644 pkg/cmd/dev/main.go create mode 100644 pkg/cmd/dev/test.go diff --git a/Makefile b/Makefile index 780d120fdf19..dd33d57404b6 100644 --- a/Makefile +++ b/Makefile @@ -1690,6 +1690,7 @@ bins = \ bin/github-pull-request-make \ bin/gossipsim \ bin/langgen \ + bin/dev \ bin/protoc-gen-gogoroach \ bin/publish-artifacts \ bin/publish-provisional-artifacts \ diff --git a/pkg/cmd/dev/BUILD.bazel b/pkg/cmd/dev/BUILD.bazel new file mode 100644 index 000000000000..dd6463deded9 --- /dev/null +++ b/pkg/cmd/dev/BUILD.bazel @@ -0,0 +1,25 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "dev_lib", + srcs = [ + "bench.go", + "build.go", + "generate.go", + "lint.go", + "main.go", + "test.go", + ], + importpath = "github.com/cockroachdb/cockroach/pkg/cmd/dev", + visibility = ["//visibility:private"], + deps = [ + "@com_github_cockroachdb_errors//:errors", + "@com_github_spf13_cobra//:cobra", + ], +) + +go_binary( + name = "dev", + embed = [":dev_lib"], + visibility = ["//visibility:public"], +) diff --git a/pkg/cmd/dev/bench.go b/pkg/cmd/dev/bench.go new file mode 100644 index 000000000000..a0107109e558 --- /dev/null +++ b/pkg/cmd/dev/bench.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +// benchCmd runs the specified cockroachdb benchmarks. +var benchCmd = &cobra.Command{ + Use: "bench", + Short: `Run the specified benchmarks`, + Long: `Run the specified benchmarks.`, + Example: ` + dev bench --pkg=sql/parser --filter=BenchmarkParse`, + Args: cobra.NoArgs, + RunE: runBench, +} + +func runBench(cmd *cobra.Command, args []string) error { + // TODO(irfansharif): Flesh out the example usage patterns. + return errors.New("unimplemented") +} diff --git a/pkg/cmd/dev/build.go b/pkg/cmd/dev/build.go new file mode 100644 index 000000000000..871ffdbbb0b9 --- /dev/null +++ b/pkg/cmd/dev/build.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +// buildCmd builds the specified binaries. +var buildCmd = &cobra.Command{ + Use: "build", + Short: "Build the specified binaries", + Long: "Build the specified binaries.", + Example: ` + dev build cockroach --tags=deadlock + dev build cockroach-{short,oss} + dev build {opt,exec}gen`, + Args: cobra.NoArgs, + RunE: runBuild, +} + +func runBuild(cmd *cobra.Command, args []string) error { + // TODO(irfansharif): Flesh out the example usage patterns. + return errors.New("unimplemented") +} diff --git a/pkg/cmd/dev/generate.go b/pkg/cmd/dev/generate.go new file mode 100644 index 000000000000..c25a81cbc851 --- /dev/null +++ b/pkg/cmd/dev/generate.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +// generateCmd generates the specified files. +var generateCmd = &cobra.Command{ + Use: "generate", + Aliases: []string{"gen"}, + Short: `Generate the specified files`, + Long: `Generate the specified files.`, + Example: ` + dev gen bazel + dev generate protobuf + dev generate {exec,opt}gen +`, + Args: cobra.NoArgs, + RunE: runGenerate, +} + +func runGenerate(cmd *cobra.Command, args []string) error { + // TODO(irfansharif): Flesh out the example usage patterns. + return errors.New("unimplemented") +} diff --git a/pkg/cmd/dev/lint.go b/pkg/cmd/dev/lint.go new file mode 100644 index 000000000000..29176067d37d --- /dev/null +++ b/pkg/cmd/dev/lint.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +// lintCmd runs the specified linters. +var lintCmd = &cobra.Command{ + Use: "lint [pkg] (flags)", + Short: `Run the specified linters`, + Long: `Run the specified linters.`, + Example: ` + dev lint --filter=TestLowercaseFunctionNames --short --timeout=1m`, + Args: cobra.NoArgs, + RunE: runLint, +} + +func runLint(cmd *cobra.Command, args []string) error { + // TODO(irfansharif): Flesh out the example usage patterns. + return errors.New("unimplemented") +} diff --git a/pkg/cmd/dev/main.go b/pkg/cmd/dev/main.go new file mode 100644 index 000000000000..712b9098d4fa --- /dev/null +++ b/pkg/cmd/dev/main.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var devCmd = &cobra.Command{ + Use: "dev [command] (flags)", + Short: "Dev is the general-purpose dev tool for folks working on cockroachdb/cockroach.", + Long: ` +Dev is the general-purpose dev tool for folks working cockroachdb/cockroach. It +lets engineers do a few things: + +- build various binaries (cockroach, optgen, ...) +- run arbitrary tests (unit tests, logic tests, ...) +- run tests under arbitrary configurations (under stress, using race builds, ...) +- generate code (bazel files, protobufs, ...) + +...and much more. + +(PS: Almost none of the above is implemented yet, haha.) +`, +} + +func init() { + devCmd.AddCommand( + benchCmd, + buildCmd, + generateCmd, + lintCmd, + testCmd, + ) + + // Hide the `help` sub-command. + devCmd.SetHelpCommand(&cobra.Command{ + Use: "noop-help", + Hidden: true, + }) +} + +func main() { + if err := devCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/pkg/cmd/dev/test.go b/pkg/cmd/dev/test.go new file mode 100644 index 000000000000..8645b79bb49a --- /dev/null +++ b/pkg/cmd/dev/test.go @@ -0,0 +1,35 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +// testCmd runs the specified cockroachdb tests. +var testCmd = &cobra.Command{ + Use: "test [pkg] (flags)", + Short: `Run the specified tests`, + Long: `Run the specified tests.`, + Example: ` + dev test kv/kvserver --filter=TestReplicaGC* -v -show-logs --timeout=1m + dev test --stress --race ... + dev test --logic --files=prepare|fk --subtests=20042 --config=local + dev test --fuzz sql/sem/tree --filter=Decimal`, + Args: cobra.NoArgs, + RunE: runTest, +} + +func runTest(cmd *cobra.Command, args []string) error { + // TODO(irfansharif): Flesh out the example usage patterns. + return errors.New("unimplemented") +} From b586ce7ace62b9d0fc3b66c175289d63d6b69b3a Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Mon, 23 Nov 2020 10:21:55 -0500 Subject: [PATCH 2/4] dev: flesh out `dev test` This is still pretty bare-bones, but better something than nothing. Release note: None --- pkg/cmd/dev/BUILD.bazel | 3 + pkg/cmd/dev/build.go | 2 +- pkg/cmd/dev/generate.go | 2 +- pkg/cmd/dev/lint.go | 2 +- pkg/cmd/dev/main.go | 39 ++++++++-- pkg/cmd/dev/test.go | 168 ++++++++++++++++++++++++++++++++++++++-- pkg/cmd/dev/util.go | 43 ++++++++++ 7 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 pkg/cmd/dev/util.go diff --git a/pkg/cmd/dev/BUILD.bazel b/pkg/cmd/dev/BUILD.bazel index dd6463deded9..e93fb19307d9 100644 --- a/pkg/cmd/dev/BUILD.bazel +++ b/pkg/cmd/dev/BUILD.bazel @@ -9,11 +9,14 @@ go_library( "lint.go", "main.go", "test.go", + "util.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/cmd/dev", visibility = ["//visibility:private"], deps = [ + "//pkg/util/log", "@com_github_cockroachdb_errors//:errors", + "@com_github_cockroachdb_redact//:redact", "@com_github_spf13_cobra//:cobra", ], ) diff --git a/pkg/cmd/dev/build.go b/pkg/cmd/dev/build.go index 871ffdbbb0b9..586144958566 100644 --- a/pkg/cmd/dev/build.go +++ b/pkg/cmd/dev/build.go @@ -17,7 +17,7 @@ import ( // buildCmd builds the specified binaries. var buildCmd = &cobra.Command{ - Use: "build", + Use: "build ", Short: "Build the specified binaries", Long: "Build the specified binaries.", Example: ` diff --git a/pkg/cmd/dev/generate.go b/pkg/cmd/dev/generate.go index c25a81cbc851..42367aabbde6 100644 --- a/pkg/cmd/dev/generate.go +++ b/pkg/cmd/dev/generate.go @@ -17,7 +17,7 @@ import ( // generateCmd generates the specified files. var generateCmd = &cobra.Command{ - Use: "generate", + Use: "generate ", Aliases: []string{"gen"}, Short: `Generate the specified files`, Long: `Generate the specified files.`, diff --git a/pkg/cmd/dev/lint.go b/pkg/cmd/dev/lint.go index 29176067d37d..8c1c9df1fe5b 100644 --- a/pkg/cmd/dev/lint.go +++ b/pkg/cmd/dev/lint.go @@ -17,7 +17,7 @@ import ( // lintCmd runs the specified linters. var lintCmd = &cobra.Command{ - Use: "lint [pkg] (flags)", + Use: "lint", Short: `Run the specified linters`, Long: `Run the specified linters.`, Example: ` diff --git a/pkg/cmd/dev/main.go b/pkg/cmd/dev/main.go index 712b9098d4fa..a6e39833636d 100644 --- a/pkg/cmd/dev/main.go +++ b/pkg/cmd/dev/main.go @@ -11,15 +11,18 @@ package main import ( - "fmt" + "context" "os" + "os/exec" + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/spf13/cobra" ) var devCmd = &cobra.Command{ - Use: "dev [command] (flags)", - Short: "Dev is the general-purpose dev tool for folks working on cockroachdb/cockroach.", + Use: "dev [command] (flags)", + Short: "Dev is the general-purpose dev tool for folks working on cockroachdb/cockroach.", + Version: "v0.0", Long: ` Dev is the general-purpose dev tool for folks working cockroachdb/cockroach. It lets engineers do a few things: @@ -33,8 +36,21 @@ lets engineers do a few things: (PS: Almost none of the above is implemented yet, haha.) `, + // Disable automatic printing of usage information whenever an error + // occurs. We presume that most errors will not the result of bad command + // invocation; they'll be due to legitimate build/test errors. Printing out + // the usage information in these cases obscures the real cause of the + // error. Commands should manually print usage information when the error + // is, in fact, a result of a bad invocation, e.g. too many arguments. + SilenceUsage: true, + // Disable automatic printing of the error. We want to also print + // details and hints, which cobra does not do for us. Instead + // we do the printing in the command implementation. + SilenceErrors: true, } +var bazel = "bazel" + func init() { devCmd.AddCommand( benchCmd, @@ -51,9 +67,22 @@ func init() { }) } -func main() { +func runDev(ctx context.Context) error { + _, err := exec.LookPath(bazel) + if err != nil { + log.Errorf(ctx, "expected to find bazel in $PATH") + return err + } + if err := devCmd.Execute(); err != nil { - fmt.Println(err) + return err + } + return nil +} + +func main() { + ctx := context.Background() + if err := runDev(ctx); err != nil { os.Exit(1) } } diff --git a/pkg/cmd/dev/test.go b/pkg/cmd/dev/test.go index 8645b79bb49a..b14820fb5667 100644 --- a/pkg/cmd/dev/test.go +++ b/pkg/cmd/dev/test.go @@ -11,25 +11,179 @@ package main import ( + "bufio" + "context" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" + "github.com/cockroachdb/redact" "github.com/spf13/cobra" ) // testCmd runs the specified cockroachdb tests. var testCmd = &cobra.Command{ - Use: "test [pkg] (flags)", + Use: "test [pkg..]", Short: `Run the specified tests`, Long: `Run the specified tests.`, Example: ` - dev test kv/kvserver --filter=TestReplicaGC* -v -show-logs --timeout=1m + dev test + dev test pkg/kv/kvserver --filter=TestReplicaGC* -v -show-logs --timeout=1m dev test --stress --race ... dev test --logic --files=prepare|fk --subtests=20042 --config=local dev test --fuzz sql/sem/tree --filter=Decimal`, - Args: cobra.NoArgs, - RunE: runTest, + Args: cobra.MinimumNArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + return runTest(ctx, cmd, args) + }, +} + +var ( + // General testing flags. + filterFlag = "filter" + timeoutFlag = "timeout" + showLogsFlag = "show-logs" + vFlag = "verbose" + stressFlag = "stress" + raceFlag = "race" + ignoreCacheFlag = "ignore-cache" + + // Fuzz testing related flag. + fuzzFlag = "fuzz" + + // Logic test related flags. + logicFlag = "logic" + filesFlag = "files" + subtestsFlag = "subtests" + configFlag = "config" +) + +func init() { + // Attach flags for the test sub-command. + testCmd.Flags().StringP(filterFlag, "f", "", "run unit tests matching this regex") + testCmd.Flags().Duration(timeoutFlag, 20*time.Minute, "timeout for test") + testCmd.Flags().Bool(showLogsFlag, false, "print logs instead of saving them in files") + testCmd.Flags().BoolP(vFlag, "v", false, "enable logging during test runs") + testCmd.Flags().Bool(stressFlag, false, "run tests under stress") + testCmd.Flags().Bool(raceFlag, false, "run tests using race builds") + testCmd.Flags().Bool(ignoreCacheFlag, false, "ignore cached test runs") + + // Fuzz testing related flag. + testCmd.Flags().Bool(fuzzFlag, false, "run fuzz tests") + + // Logic test related flags. + testCmd.Flags().Bool(logicFlag, false, "run logic tests") + testCmd.Flags().String(filesFlag, "", "run logic tests for files matching this regex") + testCmd.Flags().String(subtestsFlag, "", "run logic test subtests matching this regex") + testCmd.Flags().String(configFlag, "", "run logic tests under the specified config") +} + +func runTest(ctx context.Context, cmd *cobra.Command, pkgs []string) error { + if logicTest := mustGetFlagBool(cmd, logicFlag); logicTest { + return runLogicTest(ctx, cmd) + } + + if fuzzTest := mustGetFlagBool(cmd, fuzzFlag); fuzzTest { + return runFuzzTest(ctx, cmd, pkgs) + } + + return runUnitTest(ctx, cmd, pkgs) +} + +func runUnitTest(ctx context.Context, cmd *cobra.Command, pkgs []string) error { + stress := mustGetFlagBool(cmd, stressFlag) + race := mustGetFlagBool(cmd, raceFlag) + filter := mustGetFlagString(cmd, filterFlag) + timeout := mustGetFlagDuration(cmd, timeoutFlag) + ignoreCache := mustGetFlagBool(cmd, ignoreCacheFlag) + verbose := mustGetFlagBool(cmd, vFlag) + showLogs := mustGetFlagBool(cmd, showLogsFlag) + + if showLogs { + return errors.New("-show-logs unimplemented") + } + + log.Infof(ctx, "unit test args: stress=%t race=%t filter=%s timeout=%s ignore-cache=%t pkgs=%s", + stress, race, filter, timeout, ignoreCache, pkgs) + + var args []string + args = append(args, "test") + if race { + args = append(args, "--features", "race") + } + + args = append(args, "--color=yes") + for _, pkg := range pkgs { + if strings.HasSuffix(pkg, "...") { + args = append(args, fmt.Sprintf("@cockroach//%s", pkg)) + } else { + components := strings.Split(pkg, "/") + pkgName := components[len(components)-1] + args = append(args, fmt.Sprintf("@cockroach//%s:%s_test", pkg, pkgName)) + } + } + + if ignoreCache { + args = append(args, "--nocache_test_results") + } + if stress { + // TODO(irfansharif): Should this be pulled into a top-level flag? + // Should we just re-purpose timeout here? + args = append(args, "--run_under", fmt.Sprintf("stress -maxtime=%s", timeout)) + } + if filter != "" { + args = append(args, fmt.Sprintf("--test_filter=%s", filter)) + } + if verbose { + args = append(args, "--test_output", "all", "--test_arg", "-test.v") + } + + bazelCmd := exec.CommandContext(ctx, "bazel", args...) + + log.Infof(ctx, "executing: %s", log.Safe(bazelCmd.String())) + + stdout, err := bazelCmd.StdoutPipe() + if err != nil { + return err + } + bazelCmd.Stderr = bazelCmd.Stdout + + if err := bazelCmd.Start(); err != nil { + return err + } + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + log.Infof(ctx, "-- %s\n", redact.Safe(line)) + } + if err := scanner.Err(); err != nil { + return err + } + if err := bazelCmd.Wait(); err != nil { + return err + } + + return nil } -func runTest(cmd *cobra.Command, args []string) error { - // TODO(irfansharif): Flesh out the example usage patterns. - return errors.New("unimplemented") +func runLogicTest(ctx context.Context, cmd *cobra.Command) error { + files := mustGetFlagString(cmd, filesFlag) + subtests := mustGetFlagString(cmd, subtestsFlag) + config := mustGetFlagString(cmd, configFlag) + + log.Infof(ctx, "logic test args: files=%s subtests=%s config=%s", + files, subtests, config) + return errors.New("--logic unimplemented") +} + +func runFuzzTest(ctx context.Context, cmd *cobra.Command, pkgs []string) error { + filter := mustGetFlagString(cmd, filterFlag) + + log.Infof(ctx, "fuzz test args: filter=%s pkgs=%s", filter, pkgs) + return errors.New("--fuzz unimplemented") } diff --git a/pkg/cmd/dev/util.go b/pkg/cmd/dev/util.go new file mode 100644 index 000000000000..2969339ae82f --- /dev/null +++ b/pkg/cmd/dev/util.go @@ -0,0 +1,43 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "context" + "time" + + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/spf13/cobra" +) + +func mustGetFlagString(cmd *cobra.Command, name string) string { + val, err := cmd.Flags().GetString(name) + if err != nil { + log.Fatalf(context.Background(), "unexpected error: %v", err) + } + return val +} + +func mustGetFlagBool(cmd *cobra.Command, name string) bool { + val, err := cmd.Flags().GetBool(name) + if err != nil { + log.Fatalf(context.Background(), "unexpected error: %v", err) + } + return val +} + +func mustGetFlagDuration(cmd *cobra.Command, name string) time.Duration { + val, err := cmd.Flags().GetDuration(name) + if err != nil { + log.Fatalf(context.Background(), "unexpected error: %v", err) + } + return val +} From 69e0d79d0a0a7dffa225059103863fdfc4516c5a Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Mon, 23 Nov 2020 14:35:26 -0500 Subject: [PATCH 3/4] dev: flesh out `dev gen bazel` This is still pretty bare-bones, but better something than nothing. Release note: None --- pkg/cmd/dev/generate.go | 54 +++++++++++++++++++++++++++++++++++------ pkg/cmd/dev/test.go | 33 ++----------------------- pkg/cmd/dev/util.go | 33 +++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 39 deletions(-) diff --git a/pkg/cmd/dev/generate.go b/pkg/cmd/dev/generate.go index 42367aabbde6..567558746098 100644 --- a/pkg/cmd/dev/generate.go +++ b/pkg/cmd/dev/generate.go @@ -11,26 +11,64 @@ package main import ( + "context" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) // generateCmd generates the specified files. var generateCmd = &cobra.Command{ - Use: "generate ", + Use: "generate [target..]", Aliases: []string{"gen"}, Short: `Generate the specified files`, Long: `Generate the specified files.`, Example: ` - dev gen bazel + dev generate + dev generate bazel dev generate protobuf - dev generate {exec,opt}gen -`, - Args: cobra.NoArgs, + dev generate {exec,opt}gen`, + Args: cobra.MinimumNArgs(0), RunE: runGenerate, } -func runGenerate(cmd *cobra.Command, args []string) error { - // TODO(irfansharif): Flesh out the example usage patterns. - return errors.New("unimplemented") +// TODO(irfansharif): Flesh out the remaining targets. +type generator func(ctx context.Context, cmd *cobra.Command) error + +var generators = []generator{ + generateBazel, +} + +func runGenerate(cmd *cobra.Command, targets []string) error { + ctx := context.Background() + + if len(targets) == 0 { + // Generate all targets. + for _, gen := range generators { + if err := gen(ctx, cmd); err != nil { + return err + } + } + return nil + } + + for _, target := range targets { + var gen generator + switch target { + case "bazel": + gen = generateBazel + default: + return errors.Newf("unrecognized target: %s", target) + } + + if err := gen(ctx, cmd); err != nil { + return err + } + } + + return nil +} + +func generateBazel(ctx context.Context, cmd *cobra.Command) error { + return execute(ctx, "bazel", "run", "@cockroach//:gazelle", "--color=yes") } diff --git a/pkg/cmd/dev/test.go b/pkg/cmd/dev/test.go index b14820fb5667..419ce2735359 100644 --- a/pkg/cmd/dev/test.go +++ b/pkg/cmd/dev/test.go @@ -11,16 +11,13 @@ package main import ( - "bufio" "context" "fmt" - "os/exec" "strings" "time" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" - "github.com/cockroachdb/redact" "github.com/spf13/cobra" ) @@ -115,8 +112,8 @@ func runUnitTest(ctx context.Context, cmd *cobra.Command, pkgs []string) error { if race { args = append(args, "--features", "race") } - args = append(args, "--color=yes") + for _, pkg := range pkgs { if strings.HasSuffix(pkg, "...") { args = append(args, fmt.Sprintf("@cockroach//%s", pkg)) @@ -142,33 +139,7 @@ func runUnitTest(ctx context.Context, cmd *cobra.Command, pkgs []string) error { args = append(args, "--test_output", "all", "--test_arg", "-test.v") } - bazelCmd := exec.CommandContext(ctx, "bazel", args...) - - log.Infof(ctx, "executing: %s", log.Safe(bazelCmd.String())) - - stdout, err := bazelCmd.StdoutPipe() - if err != nil { - return err - } - bazelCmd.Stderr = bazelCmd.Stdout - - if err := bazelCmd.Start(); err != nil { - return err - } - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - line := scanner.Text() - log.Infof(ctx, "-- %s\n", redact.Safe(line)) - } - if err := scanner.Err(); err != nil { - return err - } - if err := bazelCmd.Wait(); err != nil { - return err - } - - return nil + return execute(ctx, "bazel", args...) } func runLogicTest(ctx context.Context, cmd *cobra.Command) error { diff --git a/pkg/cmd/dev/util.go b/pkg/cmd/dev/util.go index 2969339ae82f..8e6ef57ccde5 100644 --- a/pkg/cmd/dev/util.go +++ b/pkg/cmd/dev/util.go @@ -11,9 +11,13 @@ package main import ( + "bufio" "context" + "os/exec" "time" + "github.com/cockroachdb/redact" + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/spf13/cobra" ) @@ -41,3 +45,32 @@ func mustGetFlagDuration(cmd *cobra.Command, name string) time.Duration { } return val } + +func execute(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + log.Infof(ctx, "executing: %s", log.Safe(cmd.String())) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + cmd.Stderr = cmd.Stdout + + if err := cmd.Start(); err != nil { + return err + } + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + log.Infof(ctx, "-- %s\n", redact.Safe(line)) + } + if err := scanner.Err(); err != nil { + return err + } + if err := cmd.Wait(); err != nil { + return err + } + + return nil +} From a04c92943cdf238d0b225ed8a45b7cfbd5abefbe Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Mon, 23 Nov 2020 23:07:02 -0500 Subject: [PATCH 4/4] dev: flesh out `dev build ...` This is still pretty bare-bones, but better something than nothing. Release note: None --- pkg/cmd/dev/build.go | 52 +++++++++++++++++++++++++++++++++++++---- pkg/cmd/dev/generate.go | 7 +++++- pkg/cmd/dev/util.go | 3 +-- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/pkg/cmd/dev/build.go b/pkg/cmd/dev/build.go index 586144958566..0117a914171a 100644 --- a/pkg/cmd/dev/build.go +++ b/pkg/cmd/dev/build.go @@ -11,6 +11,9 @@ package main import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) @@ -20,15 +23,56 @@ var buildCmd = &cobra.Command{ Use: "build ", Short: "Build the specified binaries", Long: "Build the specified binaries.", + // TODO(irfansharif): Flesh out the example usage patterns. Example: ` dev build cockroach --tags=deadlock dev build cockroach-{short,oss} dev build {opt,exec}gen`, - Args: cobra.NoArgs, + Args: cobra.MinimumNArgs(0), RunE: runBuild, } -func runBuild(cmd *cobra.Command, args []string) error { - // TODO(irfansharif): Flesh out the example usage patterns. - return errors.New("unimplemented") +var buildTargetMapping = map[string]string{ + "cockroach": "@cockroach//pkg/cmd/cockroach", + "cockroach-oss": "@cockroach//pkg/cmd/cockroach-oss", + "cockroach-short": "@cockroach//pkg/cmd/cockroach-short", + "dev": "@cockroach//pkg/cmd/dev", + "docgen": "@cockroach//pkg/cmd/docgen", + "execgen": "@cockroach//pkg/sql/colexec/execgen/cmd/execgen", + "optgen": "@cockroach//pkg/sql/opt/optgen/cmd/optgen", + "optfmt": "@cockroach//pkg/sql/opt/optgen/cmd/optfmt", + "langgen": "@cockroach//pkg/sql/opt/optgen/cmd/langgen", + "roachprod": "@cockroach//pkg/cmd/roachprod", + "roachprod-stress": "@cockroach//pkg/cmd/roachprod-stress", + "workload": "@cockroach//pkg/cmd/workload", + "roachtest": "@cockroach//pkg/cmd/roachtest", +} + +func runBuild(cmd *cobra.Command, targets []string) error { + ctx := context.Background() + + if len(targets) == 0 { + // Default to building the cockroach binary. + targets = append(targets, "cockroach") + } + + // TODO(irfansharif): Add grouping shorthands like "all" or "bins", etc. + // TODO(irfansharif): Extract built binaries out of the bazel sandbox. + // TODO(irfansharif): Make sure all the relevant binary targets are defined + // above, and in usage docs. + + var args []string + args = append(args, "build") + args = append(args, "--color=yes") + + for _, target := range targets { + buildTarget, ok := buildTargetMapping[target] + if !ok { + log.Errorf(ctx, "unrecognized target: %s", target) + return errors.Newf("unrecognized target") + } + + args = append(args, buildTarget) + } + return execute(ctx, "bazel", args...) } diff --git a/pkg/cmd/dev/generate.go b/pkg/cmd/dev/generate.go index 567558746098..2115d912c12f 100644 --- a/pkg/cmd/dev/generate.go +++ b/pkg/cmd/dev/generate.go @@ -13,6 +13,7 @@ package main import ( "context" + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) @@ -29,6 +30,9 @@ var generateCmd = &cobra.Command{ dev generate protobuf dev generate {exec,opt}gen`, Args: cobra.MinimumNArgs(0), + // TODO(irfansharif): Errors but default just eaten up. Let's wrap these + // invocations in something that prints out the appropriate error log + // (especially considering we've SilenceErrors-ed things away). RunE: runGenerate, } @@ -58,7 +62,8 @@ func runGenerate(cmd *cobra.Command, targets []string) error { case "bazel": gen = generateBazel default: - return errors.Newf("unrecognized target: %s", target) + log.Errorf(ctx, "unrecognized target: %s", target) + return errors.Newf("unrecognized target") } if err := gen(ctx, cmd); err != nil { diff --git a/pkg/cmd/dev/util.go b/pkg/cmd/dev/util.go index 8e6ef57ccde5..1bf5ee63bcd4 100644 --- a/pkg/cmd/dev/util.go +++ b/pkg/cmd/dev/util.go @@ -16,9 +16,8 @@ import ( "os/exec" "time" - "github.com/cockroachdb/redact" - "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/redact" "github.com/spf13/cobra" )