diff --git a/go/cmd/topo2topo/plugin_consultopo.go b/go/cmd/topo2topo/cli/plugin_consultopo.go similarity index 98% rename from go/cmd/topo2topo/plugin_consultopo.go rename to go/cmd/topo2topo/cli/plugin_consultopo.go index 59d6774fdbc..a128f294a42 100644 --- a/go/cmd/topo2topo/plugin_consultopo.go +++ b/go/cmd/topo2topo/cli/plugin_consultopo.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package cli // This plugin imports consultopo to register the consul implementation of TopoServer. diff --git a/go/cmd/topo2topo/plugin_etcd2topo.go b/go/cmd/topo2topo/cli/plugin_etcd2topo.go similarity index 98% rename from go/cmd/topo2topo/plugin_etcd2topo.go rename to go/cmd/topo2topo/cli/plugin_etcd2topo.go index d99ef51d4af..5a51923cf00 100644 --- a/go/cmd/topo2topo/plugin_etcd2topo.go +++ b/go/cmd/topo2topo/cli/plugin_etcd2topo.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package cli // This plugin imports etcd2topo to register the etcd2 implementation of TopoServer. diff --git a/go/cmd/topo2topo/plugin_zk2topo.go b/go/cmd/topo2topo/cli/plugin_zk2topo.go similarity index 98% rename from go/cmd/topo2topo/plugin_zk2topo.go rename to go/cmd/topo2topo/cli/plugin_zk2topo.go index 62dda455df7..66d14988c75 100644 --- a/go/cmd/topo2topo/plugin_zk2topo.go +++ b/go/cmd/topo2topo/cli/plugin_zk2topo.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package cli import ( // Imports and register the zk2 TopologyServer diff --git a/go/cmd/topo2topo/cli/topo2topo.go b/go/cmd/topo2topo/cli/topo2topo.go new file mode 100644 index 00000000000..6e7e173872b --- /dev/null +++ b/go/cmd/topo2topo/cli/topo2topo.go @@ -0,0 +1,158 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/vt/grpccommon" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/helpers" +) + +var ( + fromImplementation string + fromServerAddress string + fromRoot string + toImplementation string + toServerAddress string + toRoot string + compare bool + doKeyspaces bool + doShards bool + doShardReplications bool + doTablets bool + doRoutingRules bool + + Main = &cobra.Command{ + Use: "topo2topo", + Short: "topo2topo copies Vitess topology data from one topo server to another.", + Long: `topo2topo copies Vitess topology data from one topo server to another. +It can also be used to compare data between two topologies.`, + Args: cobra.NoArgs, + PreRunE: servenv.CobraPreRunE, + RunE: run, + } +) + +func init() { + servenv.MoveFlagsToCobraCommand(Main) + + Main.Flags().StringVar(&fromImplementation, "from_implementation", fromImplementation, "topology implementation to copy data from") + Main.Flags().StringVar(&fromServerAddress, "from_server", fromServerAddress, "topology server address to copy data from") + Main.Flags().StringVar(&fromRoot, "from_root", fromRoot, "topology server root to copy data from") + Main.Flags().StringVar(&toImplementation, "to_implementation", toImplementation, "topology implementation to copy data to") + Main.Flags().StringVar(&toServerAddress, "to_server", toServerAddress, "topology server address to copy data to") + Main.Flags().StringVar(&toRoot, "to_root", toRoot, "topology server root to copy data to") + Main.Flags().BoolVar(&compare, "compare", compare, "compares data between topologies") + Main.Flags().BoolVar(&doKeyspaces, "do-keyspaces", doKeyspaces, "copies the keyspace information") + Main.Flags().BoolVar(&doShards, "do-shards", doShards, "copies the shard information") + Main.Flags().BoolVar(&doShardReplications, "do-shard-replications", doShardReplications, "copies the shard replication information") + Main.Flags().BoolVar(&doTablets, "do-tablets", doTablets, "copies the tablet information") + Main.Flags().BoolVar(&doRoutingRules, "do-routing-rules", doRoutingRules, "copies the routing rules") + + acl.RegisterFlags(Main.Flags()) + grpccommon.RegisterFlags(Main.Flags()) +} + +func run(cmd *cobra.Command, args []string) error { + defer logutil.Flush() + servenv.Init() + + fromTS, err := topo.OpenServer(fromImplementation, fromServerAddress, fromRoot) + if err != nil { + return fmt.Errorf("Cannot open 'from' topo %v: %w", fromImplementation, err) + } + toTS, err := topo.OpenServer(toImplementation, toServerAddress, toRoot) + if err != nil { + return fmt.Errorf("Cannot open 'to' topo %v: %w", toImplementation, err) + } + + ctx := context.Background() + + if compare { + return compareTopos(ctx, fromTS, toTS) + } + + return copyTopos(ctx, fromTS, toTS) +} + +func copyTopos(ctx context.Context, fromTS, toTS *topo.Server) error { + if doKeyspaces { + if err := helpers.CopyKeyspaces(ctx, fromTS, toTS); err != nil { + return err + } + } + if doShards { + if err := helpers.CopyShards(ctx, fromTS, toTS); err != nil { + return err + } + } + if doShardReplications { + if err := helpers.CopyShardReplications(ctx, fromTS, toTS); err != nil { + return err + } + } + if doTablets { + if err := helpers.CopyTablets(ctx, fromTS, toTS); err != nil { + return err + } + } + if doRoutingRules { + if err := helpers.CopyRoutingRules(ctx, fromTS, toTS); err != nil { + return err + } + } + + return nil +} + +func compareTopos(ctx context.Context, fromTS, toTS *topo.Server) (err error) { + if doKeyspaces { + err = helpers.CompareKeyspaces(ctx, fromTS, toTS) + if err != nil { + return fmt.Errorf("Compare keyspaces failed: %w", err) + } + } + if doShards { + err = helpers.CompareShards(ctx, fromTS, toTS) + if err != nil { + return fmt.Errorf("Compare shards failed: %w", err) + } + } + if doShardReplications { + err = helpers.CompareShardReplications(ctx, fromTS, toTS) + if err != nil { + return fmt.Errorf("Compare shard replications failed: %w", err) + } + } + if doTablets { + err = helpers.CompareTablets(ctx, fromTS, toTS) + if err != nil { + return fmt.Errorf("Compare tablets failed: %w", err) + } + } + + fmt.Println("Topologies are in sync") + return nil +} diff --git a/go/cmd/topo2topo/docgen/main.go b/go/cmd/topo2topo/docgen/main.go new file mode 100644 index 00000000000..c1d29fff086 --- /dev/null +++ b/go/cmd/topo2topo/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/topo2topo/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.Main, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/topo2topo/topo2topo.go b/go/cmd/topo2topo/topo2topo.go index 157960548b8..c1276ebf504 100644 --- a/go/cmd/topo2topo/topo2topo.go +++ b/go/cmd/topo2topo/topo2topo.go @@ -17,132 +17,15 @@ limitations under the License. package main import ( - "context" - "fmt" - "os" - - "github.com/spf13/pflag" - - "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/topo2topo/cli" "vitess.io/vitess/go/exit" - "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/helpers" -) - -var ( - fromImplementation string - fromServerAddress string - fromRoot string - toImplementation string - toServerAddress string - toRoot string - compare bool - doKeyspaces bool - doShards bool - doShardReplications bool - doTablets bool - doRoutingRules bool ) -func init() { - servenv.OnParse(func(fs *pflag.FlagSet) { - fs.StringVar(&fromImplementation, "from_implementation", fromImplementation, "topology implementation to copy data from") - fs.StringVar(&fromServerAddress, "from_server", fromServerAddress, "topology server address to copy data from") - fs.StringVar(&fromRoot, "from_root", fromRoot, "topology server root to copy data from") - fs.StringVar(&toImplementation, "to_implementation", toImplementation, "topology implementation to copy data to") - fs.StringVar(&toServerAddress, "to_server", toServerAddress, "topology server address to copy data to") - fs.StringVar(&toRoot, "to_root", toRoot, "topology server root to copy data to") - fs.BoolVar(&compare, "compare", compare, "compares data between topologies") - fs.BoolVar(&doKeyspaces, "do-keyspaces", doKeyspaces, "copies the keyspace information") - fs.BoolVar(&doShards, "do-shards", doShards, "copies the shard information") - fs.BoolVar(&doShardReplications, "do-shard-replications", doShardReplications, "copies the shard replication information") - fs.BoolVar(&doTablets, "do-tablets", doTablets, "copies the tablet information") - fs.BoolVar(&doRoutingRules, "do-routing-rules", doRoutingRules, "copies the routing rules") - - acl.RegisterFlags(fs) - }) -} - func main() { defer exit.RecoverAll() - defer logutil.Flush() - fs := pflag.NewFlagSet("topo2topo", pflag.ExitOnError) - grpccommon.RegisterFlags(fs) - log.RegisterFlags(fs) - logutil.RegisterFlags(fs) - - servenv.ParseFlags("topo2topo") - servenv.Init() - - fromTS, err := topo.OpenServer(fromImplementation, fromServerAddress, fromRoot) - if err != nil { - log.Exitf("Cannot open 'from' topo %v: %v", fromImplementation, err) - } - toTS, err := topo.OpenServer(toImplementation, toServerAddress, toRoot) - if err != nil { - log.Exitf("Cannot open 'to' topo %v: %v", toImplementation, err) - } - - ctx := context.Background() - - if compare { - compareTopos(ctx, fromTS, toTS) - return - } - copyTopos(ctx, fromTS, toTS) -} - -func copyTopos(ctx context.Context, fromTS, toTS *topo.Server) { - if doKeyspaces { - helpers.CopyKeyspaces(ctx, fromTS, toTS) - } - if doShards { - helpers.CopyShards(ctx, fromTS, toTS) - } - if doShardReplications { - helpers.CopyShardReplications(ctx, fromTS, toTS) - } - if doTablets { - helpers.CopyTablets(ctx, fromTS, toTS) - } - if doRoutingRules { - helpers.CopyRoutingRules(ctx, fromTS, toTS) - } -} - -func compareTopos(ctx context.Context, fromTS, toTS *topo.Server) { - var err error - if doKeyspaces { - err = helpers.CompareKeyspaces(ctx, fromTS, toTS) - if err != nil { - log.Exitf("Compare keyspaces failed: %v", err) - } - } - if doShards { - err = helpers.CompareShards(ctx, fromTS, toTS) - if err != nil { - log.Exitf("Compare shards failed: %v", err) - } - } - if doShardReplications { - err = helpers.CompareShardReplications(ctx, fromTS, toTS) - if err != nil { - log.Exitf("Compare shard replications failed: %v", err) - } - } - if doTablets { - err = helpers.CompareTablets(ctx, fromTS, toTS) - if err != nil { - log.Exitf("Compare tablets failed: %v", err) - } - } - if err == nil { - fmt.Println("Topologies are in sync") - os.Exit(0) + if err := cli.Main.Execute(); err != nil { + log.Exitf("%s", err) } } diff --git a/go/cmd/vtexplain/cli/vtexplain.go b/go/cmd/vtexplain/cli/vtexplain.go new file mode 100644 index 00000000000..8b0622cf8a3 --- /dev/null +++ b/go/cmd/vtexplain/cli/vtexplain.go @@ -0,0 +1,196 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "context" + "fmt" + "os" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/vtexplain" + "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + + "github.com/spf13/cobra" + + querypb "vitess.io/vitess/go/vt/proto/query" +) + +var ( + sqlFlag string + sqlFileFlag string + schemaFlag string + schemaFileFlag string + vschemaFlag string + vschemaFileFlag string + ksShardMapFlag string + ksShardMapFileFlag string + normalize bool + dbName string + plannerVersionStr string + + numShards = 2 + replicationMode = "ROW" + executionMode = "multi" + outputMode = "text" + + Main = &cobra.Command{ + Use: "vtexplain", + Short: "vtexplain is a command line tool which provides information on how Vitess plans to execute a particular query.", + Long: `vtexplain is a command line tool which provides information on how Vitess plans to execute a particular query. + +It can be used to validate queries for compatibility with Vitess. + +For a user guide that describes how to use the vtexplain tool to explain how Vitess executes a particular SQL statement, see Analyzing a SQL statement. + +## Limitations + +### The VSchema must use a keyspace name. + +VTExplain requires a keyspace name for each keyspace in an input VSchema: +` + + "```\n" + + `"keyspace_name": { + "_comment": "Keyspace definition goes here." +} +` + "```" + ` + +If no keyspace name is present, VTExplain will return the following error: +` + + "```\n" + + `ERROR: initVtgateExecutor: json: cannot unmarshal bool into Go value of type map[string]json.RawMessage +` + "```\n", + Example: "Explain how Vitess will execute the query `SELECT * FROM users` using the VSchema contained in `vschemas.json` and the database schema `schema.sql`:\n\n" + + "```\nvtexplain --vschema-file vschema.json --schema-file schema.sql --sql \"SELECT * FROM users\"\n```\n\n" + + + "Explain how the example will execute on 128 shards using Row-based replication:\n\n" + + + "```\nvtexplain -- -shards 128 --vschema-file vschema.json --schema-file schema.sql --replication-mode \"ROW\" --output-mode text --sql \"INSERT INTO users (user_id, name) VALUES(1, 'john')\"\n```\n", + Args: cobra.NoArgs, + PreRunE: servenv.CobraPreRunE, + RunE: run, + } +) + +func init() { + servenv.MoveFlagsToCobraCommand(Main) + Main.Flags().StringVar(&sqlFlag, "sql", sqlFlag, "A list of semicolon-delimited SQL commands to analyze") + Main.Flags().StringVar(&sqlFileFlag, "sql-file", sqlFileFlag, "Identifies the file that contains the SQL commands to analyze") + Main.Flags().StringVar(&schemaFlag, "schema", schemaFlag, "The SQL table schema") + Main.Flags().StringVar(&schemaFileFlag, "schema-file", schemaFileFlag, "Identifies the file that contains the SQL table schema") + Main.Flags().StringVar(&vschemaFlag, "vschema", vschemaFlag, "Identifies the VTGate routing schema") + Main.Flags().StringVar(&vschemaFileFlag, "vschema-file", vschemaFileFlag, "Identifies the VTGate routing schema file") + Main.Flags().StringVar(&ksShardMapFlag, "ks-shard-map", ksShardMapFlag, "JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace") + Main.Flags().StringVar(&ksShardMapFileFlag, "ks-shard-map-file", ksShardMapFileFlag, "File containing json blob of keyspace name -> shard name -> ShardReference object") + Main.Flags().StringVar(&replicationMode, "replication-mode", replicationMode, "The replication mode to simulate -- must be set to either ROW or STATEMENT") + Main.Flags().BoolVar(&normalize, "normalize", normalize, "Whether to enable vtgate normalization") + Main.Flags().StringVar(&dbName, "dbname", dbName, "Optional database target to override normal routing") + Main.Flags().StringVar(&plannerVersionStr, "planner-version", plannerVersionStr, "Sets the default planner to use. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right") + Main.Flags().IntVar(&numShards, "shards", numShards, "Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored.") + Main.Flags().StringVar(&executionMode, "execution-mode", executionMode, "The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc") + Main.Flags().StringVar(&outputMode, "output-mode", outputMode, "Output in human-friendly text or json") + + acl.RegisterFlags(Main.Flags()) +} + +// getFileParam returns a string containing either flag is not "", +// or the content of the file named flagFile +func getFileParam(flag, flagFile, name string, required bool) (string, error) { + if flag != "" { + if flagFile != "" { + return "", fmt.Errorf("action requires only one of %v or %v-file", name, name) + } + return flag, nil + } + + if flagFile == "" { + if required { + return "", fmt.Errorf("action requires one of %v or %v-file", name, name) + } + + return "", nil + } + data, err := os.ReadFile(flagFile) + if err != nil { + return "", fmt.Errorf("cannot read file %v: %v", flagFile, err) + } + return string(data), nil +} + +func run(cmd *cobra.Command, args []string) error { + defer logutil.Flush() + + servenv.Init() + return parseAndRun() +} + +func parseAndRun() error { + plannerVersion, _ := plancontext.PlannerNameToVersion(plannerVersionStr) + if plannerVersionStr != "" && plannerVersion != querypb.ExecuteOptions_Gen4 { + return fmt.Errorf("invalid value specified for planner-version of '%s' -- valid value is Gen4 or an empty value to use the default planner", plannerVersionStr) + } + + sql, err := getFileParam(sqlFlag, sqlFileFlag, "sql", true) + if err != nil { + return err + } + + schema, err := getFileParam(schemaFlag, schemaFileFlag, "schema", true) + if err != nil { + return err + } + + vschema, err := getFileParam(vschemaFlag, vschemaFileFlag, "vschema", true) + if err != nil { + return err + } + + ksShardMap, err := getFileParam(ksShardMapFlag, ksShardMapFileFlag, "ks-shard-map", false) + if err != nil { + return err + } + + opts := &vtexplain.Options{ + ExecutionMode: executionMode, + PlannerVersion: plannerVersion, + ReplicationMode: replicationMode, + NumShards: numShards, + Normalize: normalize, + Target: dbName, + } + + vte, err := vtexplain.Init(context.Background(), vschema, schema, ksShardMap, opts) + if err != nil { + return err + } + defer vte.Stop() + + plans, err := vte.Run(sql) + if err != nil { + return err + } + + if outputMode == "text" { + fmt.Print(vte.ExplainsAsText(plans)) + } else { + fmt.Print(vtexplain.ExplainsAsJSON(plans)) + } + + return nil +} diff --git a/go/cmd/vtexplain/docgen/main.go b/go/cmd/vtexplain/docgen/main.go new file mode 100644 index 00000000000..15ea92b53bb --- /dev/null +++ b/go/cmd/vtexplain/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/vtexplain/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.Main, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/vtexplain/vtexplain.go b/go/cmd/vtexplain/vtexplain.go index 68ceed51316..37774076382 100644 --- a/go/cmd/vtexplain/vtexplain.go +++ b/go/cmd/vtexplain/vtexplain.go @@ -17,153 +17,17 @@ limitations under the License. package main import ( - "context" "fmt" - "os" - "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/vtexplain/cli" "vitess.io/vitess/go/exit" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/vtexplain" - "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - - "github.com/spf13/pflag" - - querypb "vitess.io/vitess/go/vt/proto/query" -) - -var ( - sqlFlag string - sqlFileFlag string - schemaFlag string - schemaFileFlag string - vschemaFlag string - vschemaFileFlag string - ksShardMapFlag string - ksShardMapFileFlag string - normalize bool - dbName string - plannerVersionStr string - - numShards = 2 - replicationMode = "ROW" - executionMode = "multi" - outputMode = "text" ) -func registerFlags(fs *pflag.FlagSet) { - fs.StringVar(&sqlFlag, "sql", sqlFlag, "A list of semicolon-delimited SQL commands to analyze") - fs.StringVar(&sqlFileFlag, "sql-file", sqlFileFlag, "Identifies the file that contains the SQL commands to analyze") - fs.StringVar(&schemaFlag, "schema", schemaFlag, "The SQL table schema") - fs.StringVar(&schemaFileFlag, "schema-file", schemaFileFlag, "Identifies the file that contains the SQL table schema") - fs.StringVar(&vschemaFlag, "vschema", vschemaFlag, "Identifies the VTGate routing schema") - fs.StringVar(&vschemaFileFlag, "vschema-file", vschemaFileFlag, "Identifies the VTGate routing schema file") - fs.StringVar(&ksShardMapFlag, "ks-shard-map", ksShardMapFlag, "JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace") - fs.StringVar(&ksShardMapFileFlag, "ks-shard-map-file", ksShardMapFileFlag, "File containing json blob of keyspace name -> shard name -> ShardReference object") - fs.StringVar(&replicationMode, "replication-mode", replicationMode, "The replication mode to simulate -- must be set to either ROW or STATEMENT") - fs.BoolVar(&normalize, "normalize", normalize, "Whether to enable vtgate normalization") - fs.StringVar(&dbName, "dbname", dbName, "Optional database target to override normal routing") - fs.StringVar(&plannerVersionStr, "planner-version", plannerVersionStr, "Sets the default planner to use. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right") - fs.IntVar(&numShards, "shards", numShards, "Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored.") - fs.StringVar(&executionMode, "execution-mode", executionMode, "The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc") - fs.StringVar(&outputMode, "output-mode", outputMode, "Output in human-friendly text or json") - - acl.RegisterFlags(fs) -} - -func init() { - servenv.OnParse(registerFlags) -} - -// getFileParam returns a string containing either flag is not "", -// or the content of the file named flagFile -func getFileParam(flag, flagFile, name string, required bool) (string, error) { - if flag != "" { - if flagFile != "" { - return "", fmt.Errorf("action requires only one of %v or %v-file", name, name) - } - return flag, nil - } - - if flagFile == "" { - if required { - return "", fmt.Errorf("action requires one of %v or %v-file", name, name) - } - - return "", nil - } - data, err := os.ReadFile(flagFile) - if err != nil { - return "", fmt.Errorf("cannot read file %v: %v", flagFile, err) - } - return string(data), nil -} - func main() { defer exit.RecoverAll() - defer logutil.Flush() - servenv.ParseFlags("vtexplain") - servenv.Init() - err := parseAndRun() - if err != nil { + if err := cli.Main.Execute(); err != nil { fmt.Printf("ERROR: %s\n", err) exit.Return(1) } } - -func parseAndRun() error { - plannerVersion, _ := plancontext.PlannerNameToVersion(plannerVersionStr) - if plannerVersionStr != "" && plannerVersion != querypb.ExecuteOptions_Gen4 { - return fmt.Errorf("invalid value specified for planner-version of '%s' -- valid value is Gen4 or an empty value to use the default planner", plannerVersionStr) - } - - sql, err := getFileParam(sqlFlag, sqlFileFlag, "sql", true) - if err != nil { - return err - } - - schema, err := getFileParam(schemaFlag, schemaFileFlag, "schema", true) - if err != nil { - return err - } - - vschema, err := getFileParam(vschemaFlag, vschemaFileFlag, "vschema", true) - if err != nil { - return err - } - - ksShardMap, err := getFileParam(ksShardMapFlag, ksShardMapFileFlag, "ks-shard-map", false) - if err != nil { - return err - } - - opts := &vtexplain.Options{ - ExecutionMode: executionMode, - PlannerVersion: plannerVersion, - ReplicationMode: replicationMode, - NumShards: numShards, - Normalize: normalize, - Target: dbName, - } - - vte, err := vtexplain.Init(context.Background(), vschema, schema, ksShardMap, opts) - if err != nil { - return err - } - defer vte.Stop() - - plans, err := vte.Run(sql) - if err != nil { - return err - } - - if outputMode == "text" { - fmt.Print(vte.ExplainsAsText(plans)) - } else { - fmt.Print(vtexplain.ExplainsAsJSON(plans)) - } - - return nil -} diff --git a/go/cmd/vtgateclienttest/cli/main.go b/go/cmd/vtgateclienttest/cli/main.go new file mode 100644 index 00000000000..a30cebe418d --- /dev/null +++ b/go/cmd/vtgateclienttest/cli/main.go @@ -0,0 +1,64 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cli is the implementation of vtgateclienttest. +// This program has a chain of vtgateservice.VTGateService implementations, +// each one being responsible for one test scenario. +package cli + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/vtgateclienttest/services" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/vtgate" +) + +var Main = &cobra.Command{ + Use: "vtgateclienttest", + Short: "vtgateclienttest is a chain of vtgateservice.VTGateService implementations, each one being responsible for one test scenario.", + Args: cobra.NoArgs, + PreRunE: servenv.CobraPreRunE, + RunE: run, +} + +func init() { + servenv.RegisterDefaultFlags() + servenv.RegisterFlags() + servenv.RegisterGRPCServerFlags() + servenv.RegisterGRPCServerAuthFlags() + servenv.RegisterServiceMapFlag() + + servenv.MoveFlagsToCobraCommand(Main) + + acl.RegisterFlags(Main.Flags()) +} + +func run(cmd *cobra.Command, args []string) error { + servenv.Init() + + // The implementation chain. + servenv.OnRun(func() { + s := services.CreateServices() + for _, f := range vtgate.RegisterVTGates { + f(s) + } + }) + + servenv.RunDefault() + return nil +} diff --git a/go/cmd/vtgateclienttest/plugin_grpcvtgateservice.go b/go/cmd/vtgateclienttest/cli/plugin_grpcvtgateservice.go similarity index 98% rename from go/cmd/vtgateclienttest/plugin_grpcvtgateservice.go rename to go/cmd/vtgateclienttest/cli/plugin_grpcvtgateservice.go index 4ee159710ca..bbbc6e3039e 100644 --- a/go/cmd/vtgateclienttest/plugin_grpcvtgateservice.go +++ b/go/cmd/vtgateclienttest/cli/plugin_grpcvtgateservice.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package cli // Imports and register the gRPC vtgateservice server diff --git a/go/cmd/vtgateclienttest/docgen/main.go b/go/cmd/vtgateclienttest/docgen/main.go new file mode 100644 index 00000000000..3a18cd6feeb --- /dev/null +++ b/go/cmd/vtgateclienttest/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/vtgateclienttest/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.Main, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/vtgateclienttest/main.go b/go/cmd/vtgateclienttest/main.go index 2623ab84893..313b27de04a 100644 --- a/go/cmd/vtgateclienttest/main.go +++ b/go/cmd/vtgateclienttest/main.go @@ -14,46 +14,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package main is the implementation of vtgateclienttest. -// This program has a chain of vtgateservice.VTGateService implementations, -// each one being responsible for one test scenario. package main import ( - "github.com/spf13/pflag" - - "vitess.io/vitess/go/acl" - "vitess.io/vitess/go/cmd/vtgateclienttest/services" + "vitess.io/vitess/go/cmd/vtgateclienttest/cli" "vitess.io/vitess/go/exit" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/vtgate" + "vitess.io/vitess/go/vt/log" ) -func init() { - servenv.RegisterDefaultFlags() - servenv.RegisterFlags() - servenv.RegisterGRPCServerFlags() - servenv.RegisterGRPCServerAuthFlags() - servenv.RegisterServiceMapFlag() - - servenv.OnParse(func(fs *pflag.FlagSet) { - acl.RegisterFlags(fs) - }) -} - func main() { defer exit.Recover() - servenv.ParseFlags("vtgateclienttest") - servenv.Init() - - // The implementation chain. - servenv.OnRun(func() { - s := services.CreateServices() - for _, f := range vtgate.RegisterVTGates { - f(s) - } - }) - - servenv.RunDefault() + if err := cli.Main.Execute(); err != nil { + log.Exitf("%s", err) + } } diff --git a/go/cmd/vttestserver/data/schema/app_customer/v001__create_customer_table.sql b/go/cmd/vttestserver/cli/data/schema/app_customer/v001__create_customer_table.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/app_customer/v001__create_customer_table.sql rename to go/cmd/vttestserver/cli/data/schema/app_customer/v001__create_customer_table.sql diff --git a/go/cmd/vttestserver/data/schema/app_customer/v002__add_customer_vschema.sql b/go/cmd/vttestserver/cli/data/schema/app_customer/v002__add_customer_vschema.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/app_customer/v002__add_customer_vschema.sql rename to go/cmd/vttestserver/cli/data/schema/app_customer/v002__add_customer_vschema.sql diff --git a/go/cmd/vttestserver/data/schema/app_customer/vschema.json b/go/cmd/vttestserver/cli/data/schema/app_customer/vschema.json similarity index 100% rename from go/cmd/vttestserver/data/schema/app_customer/vschema.json rename to go/cmd/vttestserver/cli/data/schema/app_customer/vschema.json diff --git a/go/cmd/vttestserver/data/schema/test_keyspace/v001__create_test_table.sql b/go/cmd/vttestserver/cli/data/schema/test_keyspace/v001__create_test_table.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/test_keyspace/v001__create_test_table.sql rename to go/cmd/vttestserver/cli/data/schema/test_keyspace/v001__create_test_table.sql diff --git a/go/cmd/vttestserver/data/schema/test_keyspace/v002__create_hash_vindex.sql b/go/cmd/vttestserver/cli/data/schema/test_keyspace/v002__create_hash_vindex.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/test_keyspace/v002__create_hash_vindex.sql rename to go/cmd/vttestserver/cli/data/schema/test_keyspace/v002__create_hash_vindex.sql diff --git a/go/cmd/vttestserver/data/schema/test_keyspace/v003__add_table_vschema.sql b/go/cmd/vttestserver/cli/data/schema/test_keyspace/v003__add_table_vschema.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/test_keyspace/v003__add_table_vschema.sql rename to go/cmd/vttestserver/cli/data/schema/test_keyspace/v003__add_table_vschema.sql diff --git a/go/cmd/vttestserver/data/schema/test_keyspace/v004__create_test_table1.sql b/go/cmd/vttestserver/cli/data/schema/test_keyspace/v004__create_test_table1.sql similarity index 100% rename from go/cmd/vttestserver/data/schema/test_keyspace/v004__create_test_table1.sql rename to go/cmd/vttestserver/cli/data/schema/test_keyspace/v004__create_test_table1.sql diff --git a/go/cmd/vttestserver/cli/main.go b/go/cmd/vttestserver/cli/main.go new file mode 100644 index 00000000000..f9a2f16cd87 --- /dev/null +++ b/go/cmd/vttestserver/cli/main.go @@ -0,0 +1,308 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// vttestserver allows users to spawn a self-contained Vitess server for local testing/CI. +package cli + +import ( + "encoding/json" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" + "google.golang.org/protobuf/encoding/prototext" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/vttest" + + vttestpb "vitess.io/vitess/go/vt/proto/vttest" +) + +type topoFlags struct { + cells []string + keyspaces []string + shards []string + replicas int + rdonly int +} + +var ( + basePort int + config vttest.Config + doSeed bool + mycnf string + protoTopo string + seed vttest.SeedConfig + topo topoFlags +) + +func (t *topoFlags) buildTopology() (*vttestpb.VTTestTopology, error) { + topo := &vttestpb.VTTestTopology{} + topo.Cells = t.cells + + keyspaces := t.keyspaces + shardCounts := t.shards + if len(keyspaces) != len(shardCounts) { + return nil, fmt.Errorf("--keyspaces must be same length as --shards") + } + + for i := range keyspaces { + name := keyspaces[i] + numshards, err := strconv.ParseInt(shardCounts[i], 10, 32) + if err != nil { + return nil, err + } + + ks := &vttestpb.Keyspace{ + Name: name, + ReplicaCount: int32(t.replicas), + RdonlyCount: int32(t.rdonly), + } + + for _, shardname := range vttest.GetShardNames(int(numshards)) { + ks.Shards = append(ks.Shards, &vttestpb.Shard{ + Name: shardname, + }) + } + + topo.Keyspaces = append(topo.Keyspaces, ks) + } + + return topo, nil +} + +func init() { + servenv.RegisterFlags() + servenv.RegisterGRPCServerFlags() + servenv.RegisterGRPCServerAuthFlags() + servenv.RegisterServiceMapFlag() +} + +func New() (cmd *cobra.Command) { + cmd = &cobra.Command{ + Use: "vttestserver", + Short: "vttestserver allows users to spawn a self-contained Vitess server for local testing/CI.", + Args: cobra.NoArgs, + PreRunE: servenv.CobraPreRunE, + RunE: run, + } + + servenv.MoveFlagsToCobraCommand(cmd) + + cmd.Flags().IntVar(&basePort, "port", 0, + "Port to use for vtcombo. If this is 0, a random port will be chosen.") + + cmd.Flags().StringVar(&protoTopo, "proto_topo", "", + "Define the fake cluster topology as a compact text format encoded"+ + " vttest proto. See vttest.proto for more information.") + + cmd.Flags().StringVar(&config.SchemaDir, "schema_dir", "", + "Directory for initial schema files. Within this dir,"+ + " there should be a subdir for each keyspace. Within"+ + " each keyspace dir, each file is executed as SQL"+ + " after the database is created on each shard."+ + " If the directory contains a vschema.json file, it"+ + " will be used as the vschema for the V3 API.") + + cmd.Flags().StringVar(&config.DefaultSchemaDir, "default_schema_dir", "", + "Default directory for initial schema files. If no schema is found"+ + " in schema_dir, default to this location.") + + cmd.Flags().StringVar(&config.DataDir, "data_dir", "", + "Directory where the data files will be placed, defaults to a random "+ + "directory under /vt/vtdataroot") + + cmd.Flags().BoolVar(&config.OnlyMySQL, "mysql_only", false, + "If this flag is set only mysql is initialized."+ + " The rest of the vitess components are not started."+ + " Also, the output specifies the mysql unix socket"+ + " instead of the vtgate port.") + + cmd.Flags().BoolVar(&config.PersistentMode, "persistent_mode", false, + "If this flag is set, the MySQL data directory is not cleaned up"+ + " when LocalCluster.TearDown() is called. This is useful for running"+ + " vttestserver as a database container in local developer environments. Note"+ + " that db migration files (--schema_dir option) and seeding of"+ + " random data (--initialize_with_random_data option) will only run during"+ + " cluster startup if the data directory does not already exist. "+ + " Changes to VSchema are persisted across cluster restarts using a simple"+ + " watcher if the --data_dir argument is specified.") + + cmd.Flags().BoolVar(&doSeed, "initialize_with_random_data", false, + "If this flag is each table-shard will be initialized"+ + " with random data. See also the 'rng_seed' and 'min_shard_size'"+ + " and 'max_shard_size' flags.") + + cmd.Flags().IntVar(&seed.RngSeed, "rng_seed", 123, + "The random number generator seed to use when initializing"+ + " with random data (see also --initialize_with_random_data)."+ + " Multiple runs with the same seed will result with the same"+ + " initial data.") + + cmd.Flags().IntVar(&seed.MinSize, "min_table_shard_size", 1000, + "The minimum number of initial rows in a table shard. Ignored if"+ + "--initialize_with_random_data is false. The actual number is chosen"+ + " randomly.") + + cmd.Flags().IntVar(&seed.MaxSize, "max_table_shard_size", 10000, + "The maximum number of initial rows in a table shard. Ignored if"+ + "--initialize_with_random_data is false. The actual number is chosen"+ + " randomly") + + cmd.Flags().Float64Var(&seed.NullProbability, "null_probability", 0.1, + "The probability to initialize a field with 'NULL' "+ + " if --initialize_with_random_data is true. Only applies to fields"+ + " that can contain NULL values.") + + cmd.Flags().StringVar(&config.MySQLBindHost, "mysql_bind_host", "localhost", + "which host to bind vtgate mysql listener to") + + cmd.Flags().StringVar(&mycnf, "extra_my_cnf", "", + "extra files to add to the config, separated by ':'") + + cmd.Flags().StringSliceVar(&topo.cells, "cells", []string{"test"}, "Comma separated list of cells") + cmd.Flags().StringSliceVar(&topo.keyspaces, "keyspaces", []string{"test_keyspace"}, + "Comma separated list of keyspaces") + cmd.Flags().StringSliceVar(&topo.shards, "num_shards", []string{"2"}, + "Comma separated shard count (one per keyspace)") + cmd.Flags().IntVar(&topo.replicas, "replica_count", 2, + "Replica tablets per shard (includes primary)") + cmd.Flags().IntVar(&topo.rdonly, "rdonly_count", 1, + "Rdonly tablets per shard") + + cmd.Flags().StringVar(&config.Charset, "charset", "utf8mb4", "MySQL charset") + + cmd.Flags().StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right") + + cmd.Flags().StringVar(&config.SnapshotFile, "snapshot_file", "", + "A MySQL DB snapshot file") + + cmd.Flags().BoolVar(&config.EnableSystemSettings, "enable_system_settings", true, "This will enable the system settings to be changed per session at the database connection level") + + cmd.Flags().StringVar(&config.TransactionMode, "transaction_mode", "MULTI", "Transaction mode MULTI (default), SINGLE or TWOPC ") + cmd.Flags().Float64Var(&config.TransactionTimeout, "queryserver-config-transaction-timeout", 0, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") + + cmd.Flags().StringVar(&config.TabletHostName, "tablet_hostname", "localhost", "The hostname to use for the tablet otherwise it will be derived from OS' hostname") + + cmd.Flags().StringVar(&config.VSchemaDDLAuthorizedUsers, "vschema_ddl_authorized_users", "", "Comma separated list of users authorized to execute vschema ddl operations via vtgate") + + cmd.Flags().StringVar(&config.ForeignKeyMode, "foreign_key_mode", "allow", "This is to provide how to handle foreign key constraint in create/alter table. Valid values are: allow, disallow") + cmd.Flags().BoolVar(&config.EnableOnlineDDL, "enable_online_ddl", true, "Allow users to submit, review and control Online DDL") + cmd.Flags().BoolVar(&config.EnableDirectDDL, "enable_direct_ddl", true, "Allow users to submit direct DDL statements") + + // flags for using an actual topo implementation for vtcombo instead of in-memory topo. useful for test setup where an external topo server is shared across multiple vtcombo processes or other components + cmd.Flags().StringVar(&config.ExternalTopoImplementation, "external_topo_implementation", "", "the topology implementation to use for vtcombo process") + cmd.Flags().StringVar(&config.ExternalTopoGlobalServerAddress, "external_topo_global_server_address", "", "the address of the global topology server for vtcombo process") + cmd.Flags().StringVar(&config.ExternalTopoGlobalRoot, "external_topo_global_root", "", "the path of the global topology data in the global topology server for vtcombo process") + + cmd.Flags().DurationVar(&config.VtgateTabletRefreshInterval, "tablet_refresh_interval", 10*time.Second, "Interval at which vtgate refreshes tablet information from topology server.") + acl.RegisterFlags(cmd.Flags()) + + return cmd +} + +func newEnv() (env vttest.Environment, err error) { + if basePort != 0 { + if config.DataDir == "" { + env, err = vttest.NewLocalTestEnv("", basePort) + if err != nil { + return + } + } else { + env, err = vttest.NewLocalTestEnvWithDirectory("", basePort, config.DataDir) + if err != nil { + return + } + } + } + + if protoTopo == "" { + config.Topology, err = topo.buildTopology() + if err != nil { + return + } + } else { + var topology vttestpb.VTTestTopology + err = prototext.Unmarshal([]byte(protoTopo), &topology) + if err != nil { + return + } + if len(topology.Cells) == 0 { + topology.Cells = append(topology.Cells, "test") + } + config.Topology = &topology + } + + if doSeed { + config.Seed = &seed + } + + if mycnf != "" { + config.ExtraMyCnf = strings.Split(mycnf, ":") + } + + return +} + +func run(cmd *cobra.Command, args []string) error { + cluster, err := runCluster() + if err != nil { + return err + } + defer cluster.TearDown() + + servenv.Init() + + kvconf := cluster.JSONConfig() + if err := json.NewEncoder(os.Stdout).Encode(kvconf); err != nil { + return err + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + <-c + + return nil +} + +func runCluster() (cluster vttest.LocalCluster, err error) { + env, err := newEnv() + if err != nil { + return + } + + log.Infof("Starting local cluster...") + log.Infof("config: %#v", config) + cluster = vttest.LocalCluster{ + Config: config, + Env: env, + } + err = cluster.Setup() + if err != nil { + return cluster, err + } + + log.Info("Local cluster started.") + + return cluster, nil +} diff --git a/go/cmd/vttestserver/vttestserver_test.go b/go/cmd/vttestserver/cli/main_test.go similarity index 95% rename from go/cmd/vttestserver/vttestserver_test.go rename to go/cmd/vttestserver/cli/main_test.go index 226d66305be..39dc8e4ea78 100644 --- a/go/cmd/vttestserver/vttestserver_test.go +++ b/go/cmd/vttestserver/cli/main_test.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package cli import ( "context" "fmt" "io" "math/rand" - "os" "os/exec" "path" "strings" @@ -54,9 +53,8 @@ type columnVindex struct { } func TestRunsVschemaMigrations(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) cluster, err := startCluster() defer cluster.TearDown() @@ -72,9 +70,8 @@ func TestRunsVschemaMigrations(t *testing.T) { } func TestPersistentMode(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) dir := t.TempDir() @@ -135,9 +132,8 @@ func TestPersistentMode(t *testing.T) { } func TestForeignKeysAndDDLModes(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) cluster, err := startCluster("--foreign_key_mode=allow", "--enable_online_ddl=true", "--enable_direct_ddl=true") assert.NoError(t, err) @@ -190,9 +186,8 @@ func TestForeignKeysAndDDLModes(t *testing.T) { } func TestCanGetKeyspaces(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) cluster, err := startCluster() assert.NoError(t, err) @@ -202,9 +197,8 @@ func TestCanGetKeyspaces(t *testing.T) { } func TestExternalTopoServerConsul(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) // Start a single consul in the background. cmd, serverAddr := startConsul(t) @@ -228,9 +222,8 @@ func TestExternalTopoServerConsul(t *testing.T) { } func TestMtlsAuth(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) // Our test root. root := t.TempDir() @@ -270,9 +263,8 @@ func TestMtlsAuth(t *testing.T) { } func TestMtlsAuthUnauthorizedFails(t *testing.T) { - args := os.Args conf := config - defer resetFlags(args, conf) + defer resetConfig(conf) // Our test root. root := t.TempDir() @@ -322,16 +314,21 @@ var clusterKeyspaces = []string{ "app_customer", } -func startCluster(flags ...string) (vttest.LocalCluster, error) { - os.Args = []string{"vttestserver"} +func startCluster(flags ...string) (cluster vttest.LocalCluster, err error) { + args := []string{"vttestserver"} schemaDirArg := "--schema_dir=data/schema" tabletHostname := "--tablet_hostname=localhost" keyspaceArg := "--keyspaces=" + strings.Join(clusterKeyspaces, ",") numShardsArg := "--num_shards=2,2" vschemaDDLAuthorizedUsers := "--vschema_ddl_authorized_users=%" alsoLogToStderr := "--alsologtostderr" // better debugging - os.Args = append(os.Args, []string{schemaDirArg, keyspaceArg, numShardsArg, tabletHostname, vschemaDDLAuthorizedUsers, alsoLogToStderr}...) - os.Args = append(os.Args, flags...) + args = append(args, []string{schemaDirArg, keyspaceArg, numShardsArg, tabletHostname, vschemaDDLAuthorizedUsers, alsoLogToStderr}...) + args = append(args, flags...) + + if err = New().ParseFlags(args); err != nil { + return + } + return runCluster() } @@ -384,8 +381,7 @@ func assertEqual(t *testing.T, actual string, expected string, message string) { } } -func resetFlags(args []string, conf vttest.Config) { - os.Args = args +func resetConfig(conf vttest.Config) { config = conf } diff --git a/go/cmd/vttestserver/docgen/main.go b/go/cmd/vttestserver/docgen/main.go new file mode 100644 index 00000000000..61f982e2e56 --- /dev/null +++ b/go/cmd/vttestserver/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/vttestserver/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.New(), dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/vttestserver/main.go b/go/cmd/vttestserver/main.go index f5f9c6bf41c..95e63fa8019 100644 --- a/go/cmd/vttestserver/main.go +++ b/go/cmd/vttestserver/main.go @@ -14,293 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -// vttestserver allows users to spawn a self-contained Vitess server for local testing/CI. package main import ( - "encoding/json" - "fmt" - "os" - "os/signal" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "github.com/spf13/pflag" - "google.golang.org/protobuf/encoding/prototext" - - "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/vttestserver/cli" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/vttest" - - vttestpb "vitess.io/vitess/go/vt/proto/vttest" ) -type topoFlags struct { - cells []string - keyspaces []string - shards []string - replicas int - rdonly int -} - -var ( - basePort int - config vttest.Config - doSeed bool - mycnf string - protoTopo string - seed vttest.SeedConfig - topo topoFlags -) - -func registerFlags(fs *pflag.FlagSet) { - fs.IntVar(&basePort, "port", 0, - "Port to use for vtcombo. If this is 0, a random port will be chosen.") - - fs.StringVar(&protoTopo, "proto_topo", "", - "Define the fake cluster topology as a compact text format encoded"+ - " vttest proto. See vttest.proto for more information.") - - fs.StringVar(&config.SchemaDir, "schema_dir", "", - "Directory for initial schema files. Within this dir,"+ - " there should be a subdir for each keyspace. Within"+ - " each keyspace dir, each file is executed as SQL"+ - " after the database is created on each shard."+ - " If the directory contains a vschema.json file, it"+ - " will be used as the vschema for the V3 API.") - - fs.StringVar(&config.DefaultSchemaDir, "default_schema_dir", "", - "Default directory for initial schema files. If no schema is found"+ - " in schema_dir, default to this location.") - - fs.StringVar(&config.DataDir, "data_dir", "", - "Directory where the data files will be placed, defaults to a random "+ - "directory under /vt/vtdataroot") - - fs.BoolVar(&config.OnlyMySQL, "mysql_only", false, - "If this flag is set only mysql is initialized."+ - " The rest of the vitess components are not started."+ - " Also, the output specifies the mysql unix socket"+ - " instead of the vtgate port.") - - fs.BoolVar(&config.PersistentMode, "persistent_mode", false, - "If this flag is set, the MySQL data directory is not cleaned up"+ - " when LocalCluster.TearDown() is called. This is useful for running"+ - " vttestserver as a database container in local developer environments. Note"+ - " that db migration files (--schema_dir option) and seeding of"+ - " random data (--initialize_with_random_data option) will only run during"+ - " cluster startup if the data directory does not already exist. "+ - " Changes to VSchema are persisted across cluster restarts using a simple"+ - " watcher if the --data_dir argument is specified.") - - fs.BoolVar(&doSeed, "initialize_with_random_data", false, - "If this flag is each table-shard will be initialized"+ - " with random data. See also the 'rng_seed' and 'min_shard_size'"+ - " and 'max_shard_size' flags.") - - fs.IntVar(&seed.RngSeed, "rng_seed", 123, - "The random number generator seed to use when initializing"+ - " with random data (see also --initialize_with_random_data)."+ - " Multiple runs with the same seed will result with the same"+ - " initial data.") - - fs.IntVar(&seed.MinSize, "min_table_shard_size", 1000, - "The minimum number of initial rows in a table shard. Ignored if"+ - "--initialize_with_random_data is false. The actual number is chosen"+ - " randomly.") - - fs.IntVar(&seed.MaxSize, "max_table_shard_size", 10000, - "The maximum number of initial rows in a table shard. Ignored if"+ - "--initialize_with_random_data is false. The actual number is chosen"+ - " randomly") - - fs.Float64Var(&seed.NullProbability, "null_probability", 0.1, - "The probability to initialize a field with 'NULL' "+ - " if --initialize_with_random_data is true. Only applies to fields"+ - " that can contain NULL values.") - - fs.StringVar(&config.MySQLBindHost, "mysql_bind_host", "localhost", - "which host to bind vtgate mysql listener to") - - fs.StringVar(&mycnf, "extra_my_cnf", "", - "extra files to add to the config, separated by ':'") - - fs.StringSliceVar(&topo.cells, "cells", []string{"test"}, "Comma separated list of cells") - fs.StringSliceVar(&topo.keyspaces, "keyspaces", []string{"test_keyspace"}, - "Comma separated list of keyspaces") - fs.StringSliceVar(&topo.shards, "num_shards", []string{"2"}, - "Comma separated shard count (one per keyspace)") - fs.IntVar(&topo.replicas, "replica_count", 2, - "Replica tablets per shard (includes primary)") - fs.IntVar(&topo.rdonly, "rdonly_count", 1, - "Rdonly tablets per shard") - - fs.StringVar(&config.Charset, "charset", "utf8mb4", "MySQL charset") - - fs.StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right") - - fs.StringVar(&config.SnapshotFile, "snapshot_file", "", - "A MySQL DB snapshot file") - - fs.BoolVar(&config.EnableSystemSettings, "enable_system_settings", true, "This will enable the system settings to be changed per session at the database connection level") - - fs.StringVar(&config.TransactionMode, "transaction_mode", "MULTI", "Transaction mode MULTI (default), SINGLE or TWOPC ") - fs.Float64Var(&config.TransactionTimeout, "queryserver-config-transaction-timeout", 0, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") - - fs.StringVar(&config.TabletHostName, "tablet_hostname", "localhost", "The hostname to use for the tablet otherwise it will be derived from OS' hostname") - - fs.StringVar(&config.VSchemaDDLAuthorizedUsers, "vschema_ddl_authorized_users", "", "Comma separated list of users authorized to execute vschema ddl operations via vtgate") - - fs.StringVar(&config.ForeignKeyMode, "foreign_key_mode", "allow", "This is to provide how to handle foreign key constraint in create/alter table. Valid values are: allow, disallow") - fs.BoolVar(&config.EnableOnlineDDL, "enable_online_ddl", true, "Allow users to submit, review and control Online DDL") - fs.BoolVar(&config.EnableDirectDDL, "enable_direct_ddl", true, "Allow users to submit direct DDL statements") - - // flags for using an actual topo implementation for vtcombo instead of in-memory topo. useful for test setup where an external topo server is shared across multiple vtcombo processes or other components - fs.StringVar(&config.ExternalTopoImplementation, "external_topo_implementation", "", "the topology implementation to use for vtcombo process") - fs.StringVar(&config.ExternalTopoGlobalServerAddress, "external_topo_global_server_address", "", "the address of the global topology server for vtcombo process") - fs.StringVar(&config.ExternalTopoGlobalRoot, "external_topo_global_root", "", "the path of the global topology data in the global topology server for vtcombo process") - - fs.DurationVar(&config.VtgateTabletRefreshInterval, "tablet_refresh_interval", 10*time.Second, "Interval at which vtgate refreshes tablet information from topology server.") - acl.RegisterFlags(fs) -} - -func init() { - servenv.OnParseFor("vttestserver", registerFlags) -} - -func (t *topoFlags) buildTopology() (*vttestpb.VTTestTopology, error) { - topo := &vttestpb.VTTestTopology{} - topo.Cells = t.cells - - keyspaces := t.keyspaces - shardCounts := t.shards - if len(keyspaces) != len(shardCounts) { - return nil, fmt.Errorf("--keyspaces must be same length as --shards") - } - - for i := range keyspaces { - name := keyspaces[i] - numshards, err := strconv.ParseInt(shardCounts[i], 10, 32) - if err != nil { - return nil, err - } - - ks := &vttestpb.Keyspace{ - Name: name, - ReplicaCount: int32(t.replicas), - RdonlyCount: int32(t.rdonly), - } - - for _, shardname := range vttest.GetShardNames(int(numshards)) { - ks.Shards = append(ks.Shards, &vttestpb.Shard{ - Name: shardname, - }) - } - - topo.Keyspaces = append(topo.Keyspaces, ks) - } - - return topo, nil -} - -// Annoying, but in unit tests, parseFlags gets called multiple times per process -// (anytime startCluster is called), so we need to guard against the second test -// to run failing with, for example: -// -// flag redefined: log_rotate_max_size -var flagsOnce sync.Once - -func parseFlags() (env vttest.Environment, err error) { - flagsOnce.Do(func() { - servenv.RegisterFlags() - servenv.RegisterGRPCServerFlags() - servenv.RegisterGRPCServerAuthFlags() - servenv.RegisterServiceMapFlag() - }) - - servenv.ParseFlags("vttestserver") - - if basePort != 0 { - if config.DataDir == "" { - env, err = vttest.NewLocalTestEnv("", basePort) - if err != nil { - return - } - } else { - env, err = vttest.NewLocalTestEnvWithDirectory("", basePort, config.DataDir) - if err != nil { - return - } - } - } - - if protoTopo == "" { - config.Topology, err = topo.buildTopology() - if err != nil { - return - } - } else { - var topology vttestpb.VTTestTopology - err = prototext.Unmarshal([]byte(protoTopo), &topology) - if err != nil { - return - } - if len(topology.Cells) == 0 { - topology.Cells = append(topology.Cells, "test") - } - config.Topology = &topology - } - - if doSeed { - config.Seed = &seed - } - - if mycnf != "" { - config.ExtraMyCnf = strings.Split(mycnf, ":") - } - - return -} - func main() { - cluster, err := runCluster() - servenv.Init() - if err != nil { + if err := cli.New().Execute(); err != nil { log.Fatal(err) } - defer cluster.TearDown() - - kvconf := cluster.JSONConfig() - if err := json.NewEncoder(os.Stdout).Encode(kvconf); err != nil { - log.Fatal(err) - } - - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - <-c -} - -func runCluster() (vttest.LocalCluster, error) { - env, err := parseFlags() - if err != nil { - log.Fatal(err) - } - log.Infof("Starting local cluster...") - log.Infof("config: %#v", config) - cluster := vttest.LocalCluster{ - Config: config, - Env: env, - } - err = cluster.Setup() - if err != nil { - return cluster, err - } - - log.Info("Local cluster started.") - - return cluster, nil } diff --git a/go/cmd/vttlstest/cli/vttlstest.go b/go/cmd/vttlstest/cli/vttlstest.go new file mode 100644 index 00000000000..4e0f9c2b95e --- /dev/null +++ b/go/cmd/vttlstest/cli/vttlstest.go @@ -0,0 +1,135 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/vt/tlstest" +) + +var ( + root = "." + parent = "ca" + serial = "01" + commonName string + + Root = &cobra.Command{ + Use: "vttlstest", + Short: "vttlstest is a tool for generating test certificates, keys, and related artifacts for TLS tests.", + Long: "vttlstest is a tool for generating test certificates, keys, and related artifacts for TLS tests.", + } + + createCACmd = &cobra.Command{ + Use: "CreateCA [--root ]", + DisableFlagsInUseLine: true, + Example: "CreateCA --root /tmp", + Short: "Create certificate authority", + Long: "Create certificate authority", + Args: cobra.NoArgs, + Run: runCreateCA, + } + + createIntermediateCACmd = &cobra.Command{ + Use: "CreateIntermediateCA [--root ] [--parent ] [--serial ] [--common-name ] ", + DisableFlagsInUseLine: true, + Example: "CreateIntermediateCA --root /tmp --parent ca mail.mycoolsite.com", + Short: "Create intermediate certificate authority", + Long: "Create intermediate certificate authority", + Args: cobra.ExactArgs(1), + Run: runCreateIntermediateCA, + } + + createCRLCmd = &cobra.Command{ + Use: "CreateCRL [--root ] ", + DisableFlagsInUseLine: true, + Example: "CreateCRL --root /tmp mail.mycoolsite.com", + Short: "Create certificate revocation list", + Long: "Create certificate revocation list", + Args: cobra.ExactArgs(1), + Run: runCreateCRL, + } + + createSignedCertCmd = &cobra.Command{ + Use: "CreateSignedCert [--root ] [--parent ] [--serial ] [--common-name ] ", + DisableFlagsInUseLine: true, + Example: "CreateSignedCert --root /tmp --common-name mail.mysite.com --parent mail.mycoolsite.com postman1", + Short: "Create signed certificate", + Long: "Create signed certificate", + Args: cobra.ExactArgs(1), + Run: runCreateSignedCert, + } + + revokeCertCmd = &cobra.Command{ + Use: "RevokeCert [--root ] [--parent ] ", + DisableFlagsInUseLine: true, + Example: "RevokeCert --root /tmp --parent mail.mycoolsite.com postman1", + Short: "Revoke a certificate", + Long: "Revoke a certificate", + Args: cobra.ExactArgs(1), + Run: runRevokeCert, + } +) + +func init() { + Root.PersistentFlags().StringVar(&root, "root", root, "root directory for all artifacts") + + Root.AddCommand(createCACmd) + Root.AddCommand(createIntermediateCACmd) + Root.AddCommand(createCRLCmd) + Root.AddCommand(createSignedCertCmd) + Root.AddCommand(revokeCertCmd) + + for _, cmd := range []*cobra.Command{createIntermediateCACmd, createSignedCertCmd} { + cmd.Flags().StringVar(&parent, "parent", parent, "Parent cert name to use. Use 'ca' for the toplevel CA.") + cmd.Flags().StringVar(&serial, "serial", serial, "Serial number for the certificate to create. Should be different for two certificates with the same parent.") + cmd.Flags().StringVar(&commonName, "common-name", commonName, "Common name for the certificate. If empty, uses the name.") + } + revokeCertCmd.Flags().StringVar(&parent, "parent", parent, "Parent cert name to use. Use 'ca' for the toplevel CA.") +} + +func runCreateCA(cmd *cobra.Command, args []string) { + tlstest.CreateCA(root) +} + +func runCreateIntermediateCA(cmd *cobra.Command, args []string) { + name := args[0] + if commonName == "" { + commonName = name + } + + tlstest.CreateIntermediateCA(root, parent, serial, name, commonName) +} + +func runCreateCRL(cmd *cobra.Command, args []string) { + ca := args[0] + tlstest.CreateCRL(root, ca) +} + +func runCreateSignedCert(cmd *cobra.Command, args []string) { + name := args[0] + if commonName == "" { + commonName = name + } + + tlstest.CreateSignedCert(root, parent, serial, name, commonName) +} + +func runRevokeCert(cmd *cobra.Command, args []string) { + name := args[0] + tlstest.RevokeCertAndRegenerateCRL(root, parent, name) +} diff --git a/go/cmd/vttlstest/docgen/main.go b/go/cmd/vttlstest/docgen/main.go new file mode 100644 index 00000000000..2354dceb493 --- /dev/null +++ b/go/cmd/vttlstest/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/vttlstest/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.Root, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/vttlstest/vttlstest.go b/go/cmd/vttlstest/vttlstest.go index 78bffb813a3..08e994c096d 100644 --- a/go/cmd/vttlstest/vttlstest.go +++ b/go/cmd/vttlstest/vttlstest.go @@ -19,126 +19,14 @@ package main import ( "github.com/spf13/cobra" + "vitess.io/vitess/go/cmd/vttlstest/cli" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/tlstest" ) -var ( - root = "." - parent = "ca" - serial = "01" - commonName string - - rootCmd = &cobra.Command{ - Use: "vttlstest", - Short: "vttlstest is a tool for generating test certificates, keys, and related artifacts for TLS tests.", - Long: "vttlstest is a tool for generating test certificates, keys, and related artifacts for TLS tests.", - } - - createCACmd = &cobra.Command{ - Use: "CreateCA [--root ]", - DisableFlagsInUseLine: true, - Example: "CreateCA --root /tmp", - Short: "Create certificate authority", - Long: "Create certificate authority", - Args: cobra.NoArgs, - Run: runCreateCA, - } - - createIntermediateCACmd = &cobra.Command{ - Use: "CreateIntermediateCA [--root ] [--parent ] [--serial ] [--common-name ] ", - DisableFlagsInUseLine: true, - Example: "CreateIntermediateCA --root /tmp --parent ca mail.mycoolsite.com", - Short: "Create intermediate certificate authority", - Long: "Create intermediate certificate authority", - Args: cobra.ExactArgs(1), - Run: runCreateIntermediateCA, - } - - createCRLCmd = &cobra.Command{ - Use: "CreateCRL [--root ] ", - DisableFlagsInUseLine: true, - Example: "CreateCRL --root /tmp mail.mycoolsite.com", - Short: "Create certificate revocation list", - Long: "Create certificate revocation list", - Args: cobra.ExactArgs(1), - Run: runCreateCRL, - } - - createSignedCertCmd = &cobra.Command{ - Use: "CreateSignedCert [--root ] [--parent ] [--serial ] [--common-name ] ", - DisableFlagsInUseLine: true, - Example: "CreateSignedCert --root /tmp --common-name mail.mysite.com --parent mail.mycoolsite.com postman1", - Short: "Create signed certificate", - Long: "Create signed certificate", - Args: cobra.ExactArgs(1), - Run: runCreateSignedCert, - } - - revokeCertCmd = &cobra.Command{ - Use: "RevokeCert [--root ] [--parent ] ", - DisableFlagsInUseLine: true, - Example: "RevokeCert --root /tmp --parent mail.mycoolsite.com postman1", - Short: "Revoke a certificate", - Long: "Revoke a certificate", - Args: cobra.ExactArgs(1), - Run: runRevokeCert, - } -) - -func init() { - rootCmd.PersistentFlags().StringVar(&root, "root", root, "root directory for all artifacts") - - rootCmd.AddCommand(createCACmd) - rootCmd.AddCommand(createIntermediateCACmd) - rootCmd.AddCommand(createCRLCmd) - rootCmd.AddCommand(createSignedCertCmd) - rootCmd.AddCommand(revokeCertCmd) - - for _, cmd := range []*cobra.Command{createIntermediateCACmd, createSignedCertCmd} { - cmd.Flags().StringVar(&parent, "parent", parent, "Parent cert name to use. Use 'ca' for the toplevel CA.") - cmd.Flags().StringVar(&serial, "serial", serial, "Serial number for the certificate to create. Should be different for two certificates with the same parent.") - cmd.Flags().StringVar(&commonName, "common-name", commonName, "Common name for the certificate. If empty, uses the name.") - } - revokeCertCmd.Flags().StringVar(&parent, "parent", parent, "Parent cert name to use. Use 'ca' for the toplevel CA.") -} - -func runCreateCA(cmd *cobra.Command, args []string) { - tlstest.CreateCA(root) -} - -func runCreateIntermediateCA(cmd *cobra.Command, args []string) { - name := args[0] - if commonName == "" { - commonName = name - } - - tlstest.CreateIntermediateCA(root, parent, serial, name, commonName) -} - -func runCreateCRL(cmd *cobra.Command, args []string) { - ca := args[0] - tlstest.CreateCRL(root, ca) -} - -func runCreateSignedCert(cmd *cobra.Command, args []string) { - name := args[0] - if commonName == "" { - commonName = name - } - - tlstest.CreateSignedCert(root, parent, serial, name, commonName) -} - -func runRevokeCert(cmd *cobra.Command, args []string) { - name := args[0] - tlstest.RevokeCertAndRegenerateCRL(root, parent, name) -} - func main() { defer exit.Recover() defer logutil.Flush() - cobra.CheckErr(rootCmd.Execute()) + cobra.CheckErr(cli.Root.Execute()) } diff --git a/go/flags/endtoend/flags_test.go b/go/flags/endtoend/flags_test.go index 713d305fc61..25cca54caf9 100644 --- a/go/flags/endtoend/flags_test.go +++ b/go/flags/endtoend/flags_test.go @@ -41,6 +41,9 @@ var ( //go:embed mysqlctld.txt mysqlctldTxt string + //go:embed topo2topo.txt + topo2topoTxt string + //go:embed vtaclcheck.txt vtaclcheckTxt string @@ -71,6 +74,9 @@ var ( //go:embed vtctldclient.txt vtctldclientTxt string + //go:embed vtgateclienttest.txt + vtgateclienttestTxt string + //go:embed vttestserver.txt vttestserverTxt string @@ -87,23 +93,25 @@ var ( zkTxt string helpOutput = map[string]string{ - "mysqlctl": mysqlctlTxt, - "mysqlctld": mysqlctldTxt, - "vtaclcheck": vtaclcheckTxt, - "vtbackup": vtbackupTxt, - "vtcombo": vtcomboTxt, - "vtctld": vtctldTxt, - "vtctlclient": vtctlclientTxt, - "vtctldclient": vtctldclientTxt, - "vtexplain": vtexplainTxt, - "vtgate": vtgateTxt, - "vtorc": vtorcTxt, - "vttablet": vttabletTxt, - "vttestserver": vttestserverTxt, - "vttlstest": vttlstestTxt, - "zk": zkTxt, - "zkctl": zkctlTxt, - "zkctld": zkctldTxt, + "mysqlctl": mysqlctlTxt, + "mysqlctld": mysqlctldTxt, + "topo2topo": topo2topoTxt, + "vtaclcheck": vtaclcheckTxt, + "vtbackup": vtbackupTxt, + "vtcombo": vtcomboTxt, + "vtctlclient": vtctlclientTxt, + "vtctld": vtctldTxt, + "vtctldclient": vtctldclientTxt, + "vtexplain": vtexplainTxt, + "vtgate": vtgateTxt, + "vtgateclienttest": vtgateclienttestTxt, + "vtorc": vtorcTxt, + "vttablet": vttabletTxt, + "vttestserver": vttestserverTxt, + "vttlstest": vttlstestTxt, + "zk": zkTxt, + "zkctl": zkctlTxt, + "zkctld": zkctldTxt, } ) diff --git a/go/flags/endtoend/topo2topo.txt b/go/flags/endtoend/topo2topo.txt new file mode 100644 index 00000000000..4391a32a1a8 --- /dev/null +++ b/go/flags/endtoend/topo2topo.txt @@ -0,0 +1,44 @@ +topo2topo copies Vitess topology data from one topo server to another. +It can also be used to compare data between two topologies. + +Usage: + topo2topo [flags] + +Flags: + --alsologtostderr log to standard error as well as files + --compare compares data between topologies + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --do-keyspaces copies the keyspace information + --do-routing-rules copies the routing rules + --do-shard-replications copies the shard replication information + --do-shards copies the shard information + --do-tablets copies the tablet information + --from_implementation string topology implementation to copy data from + --from_root string topology server root to copy data from + --from_server string topology server address to copy data from + --grpc_enable_tracing Enable gRPC tracing. + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help help for topo2topo + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --to_implementation string topology implementation to copy data to + --to_root string topology server root to copy data to + --to_server string topology server address to copy data to + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index 39adec5467c..f75559474c0 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -1,4 +1,43 @@ -Usage of vtexplain: +vtexplain is a command line tool which provides information on how Vitess plans to execute a particular query. + +It can be used to validate queries for compatibility with Vitess. + +For a user guide that describes how to use the vtexplain tool to explain how Vitess executes a particular SQL statement, see Analyzing a SQL statement. + +## Limitations + +### The VSchema must use a keyspace name. + +VTExplain requires a keyspace name for each keyspace in an input VSchema: +``` +"keyspace_name": { + "_comment": "Keyspace definition goes here." +} +``` + +If no keyspace name is present, VTExplain will return the following error: +``` +ERROR: initVtgateExecutor: json: cannot unmarshal bool into Go value of type map[string]json.RawMessage +``` + +Usage: + vtexplain [flags] + +Examples: +Explain how Vitess will execute the query `SELECT * FROM users` using the VSchema contained in `vschemas.json` and the database schema `schema.sql`: + +``` +vtexplain --vschema-file vschema.json --schema-file schema.sql --sql "SELECT * FROM users" +``` + +Explain how the example will execute on 128 shards using Row-based replication: + +``` +vtexplain -- -shards 128 --vschema-file vschema.json --schema-file schema.sql --replication-mode "ROW" --output-mode text --sql "INSERT INTO users (user_id, name) VALUES(1, 'john')" +``` + + +Flags: --alsologtostderr log to standard error as well as files --batch-interval duration Interval between logical time slots. (default 10ms) --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. @@ -10,7 +49,7 @@ Usage of vtexplain: --dbname string Optional database target to override normal routing --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) --execution-mode string The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc (default "multi") - -h, --help display usage and exit + -h, --help help for vtexplain --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) --ks-shard-map string JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace diff --git a/go/flags/endtoend/vtgateclienttest.txt b/go/flags/endtoend/vtgateclienttest.txt new file mode 100644 index 00000000000..778dbfc8243 --- /dev/null +++ b/go/flags/endtoend/vtgateclienttest.txt @@ -0,0 +1,65 @@ +vtgateclienttest is a chain of vtgateservice.VTGateService implementations, each one being responsible for one test scenario. + +Usage: + vtgateclienttest [flags] + +Flags: + --alsologtostderr log to standard error as well as files + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) + --grpc_auth_mode string Which auth plugin implementation to use (eg: static) + --grpc_auth_mtls_allowed_substrings string List of substrings of at least one of the client certificate names (separated by colon). + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_auth_static_password_file string JSON File to read the users/passwords from. + --grpc_ca string server CA to use for gRPC connections, requires TLS, and enforces client certificate check + --grpc_cert string server certificate to use for gRPC connections, requires grpc_key, enables TLS + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_crl string path to a certificate revocation list in PEM format, client certificates will be further verified against this file during TLS handshake + --grpc_enable_optional_tls enable optional TLS mode when a server accepts both TLS and plain-text connections on the same port + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_key string server private key to use for gRPC connections, requires grpc_cert, enables TLS + --grpc_max_connection_age duration Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s) + --grpc_max_connection_age_grace duration Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_port int Port to listen on for gRPC calls. If zero, do not listen. + --grpc_prometheus Enable gRPC monitoring with Prometheus. + --grpc_server_ca string path to server CA in PEM format, which will be combine with server cert, return full certificate chain to clients + --grpc_server_initial_conn_window_size int gRPC server initial connection window size + --grpc_server_initial_window_size int gRPC server initial window size + --grpc_server_keepalive_enforcement_policy_min_time duration gRPC server minimum keepalive time (default 10s) + --grpc_server_keepalive_enforcement_policy_permit_without_stream gRPC server permit client keepalive pings even when there are no active streams (RPCs) + -h, --help help for vtgateclienttest + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --port int port for the server + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vschema_ddl_authorized_users string List of users authorized to execute vschema ddl operations, or '%' to allow all users. diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 5849f0c1e81..d9fbe1618e6 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -1,4 +1,9 @@ -Usage of vttestserver: +vttestserver allows users to spawn a self-contained Vitess server for local testing/CI. + +Usage: + vttestserver [flags] + +Flags: --alsologtostderr log to standard error as well as files --app_idle_timeout duration Idle timeout for app connections (default 1m0s) --app_pool_size int Size of the connection pool for app connections (default 40) @@ -63,7 +68,7 @@ Usage of vttestserver: --grpc_server_initial_window_size int gRPC server initial window size --grpc_server_keepalive_enforcement_policy_min_time duration gRPC server minimum keepalive time (default 10s) --grpc_server_keepalive_enforcement_policy_permit_without_stream gRPC server permit client keepalive pings even when there are no active streams (RPCs) - -h, --help display usage and exit + -h, --help help for vttestserver --initialize_with_random_data If this flag is each table-shard will be initialized with random data. See also the 'rng_seed' and 'min_shard_size' and 'max_shard_size' flags. --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 27d39179688..0df706eba31 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -20,6 +20,7 @@ package helpers import ( "context" + "fmt" "google.golang.org/protobuf/proto" @@ -32,17 +33,17 @@ import ( ) // CopyKeyspaces will create the keyspaces in the destination topo. -func CopyKeyspaces(ctx context.Context, fromTS, toTS *topo.Server) { +func CopyKeyspaces(ctx context.Context, fromTS, toTS *topo.Server) error { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { - log.Fatalf("GetKeyspaces: %v", err) + return fmt.Errorf("GetKeyspaces: %w", err) } for _, keyspace := range keyspaces { ki, err := fromTS.GetKeyspace(ctx, keyspace) if err != nil { - log.Fatalf("GetKeyspace(%v): %v", keyspace, err) + return fmt.Errorf("GetKeyspace(%v): %w", keyspace, err) } if err := toTS.CreateKeyspace(ctx, keyspace, ki.Keyspace); err != nil { @@ -70,64 +71,67 @@ func CopyKeyspaces(ctx context.Context, fromTS, toTS *topo.Server) { log.Errorf("GetVSchema(%v): %v", keyspace, err) } } + + return nil } // CopyShards will create the shards in the destination topo. -func CopyShards(ctx context.Context, fromTS, toTS *topo.Server) { +func CopyShards(ctx context.Context, fromTS, toTS *topo.Server) error { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { - log.Fatalf("fromTS.GetKeyspaces: %v", err) + return fmt.Errorf("fromTS.GetKeyspaces: %w", err) } for _, keyspace := range keyspaces { shards, err := fromTS.GetShardNames(ctx, keyspace) if err != nil { - log.Fatalf("GetShardNames(%v): %v", keyspace, err) - return + return fmt.Errorf("GetShardNames(%v): %w", keyspace, err) } for _, shard := range shards { si, err := fromTS.GetShard(ctx, keyspace, shard) if err != nil { - log.Fatalf("GetShard(%v, %v): %v", keyspace, shard, err) + return fmt.Errorf("GetShard(%v, %v): %w", keyspace, shard, err) } if err := toTS.CreateShard(ctx, keyspace, shard); err != nil { if topo.IsErrType(err, topo.NodeExists) { log.Warningf("shard %v/%v already exists", keyspace, shard) } else { - log.Fatalf("CreateShard(%v, %v): %v", keyspace, shard, err) + return fmt.Errorf("CreateShard(%v, %v): %w", keyspace, shard, err) } } if _, err := toTS.UpdateShardFields(ctx, keyspace, shard, func(toSI *topo.ShardInfo) error { toSI.Shard = si.Shard.CloneVT() return nil }); err != nil { - log.Fatalf("UpdateShardFields(%v, %v): %v", keyspace, shard, err) + return fmt.Errorf("UpdateShardFields(%v, %v): %w", keyspace, shard, err) } } } + + return nil } // CopyTablets will create the tablets in the destination topo. -func CopyTablets(ctx context.Context, fromTS, toTS *topo.Server) { +func CopyTablets(ctx context.Context, fromTS, toTS *topo.Server) error { cells, err := fromTS.GetKnownCells(ctx) if err != nil { - log.Fatalf("fromTS.GetKnownCells: %v", err) + return fmt.Errorf("fromTS.GetKnownCells: %w", err) } for _, cell := range cells { tabletAliases, err := fromTS.GetTabletAliasesByCell(ctx, cell) if err != nil { - log.Fatalf("GetTabletsByCell(%v): %v", cell, err) + return fmt.Errorf("GetTabletsByCell(%v): %w", cell, err) } else { for _, tabletAlias := range tabletAliases { // read the source tablet ti, err := fromTS.GetTablet(ctx, tabletAlias) if err != nil { - log.Fatalf("GetTablet(%v): %v", tabletAlias, err) + return fmt.Errorf("GetTablet(%v): %w", tabletAlias, err) } // try to create the destination @@ -141,37 +145,39 @@ func CopyTablets(ctx context.Context, fromTS, toTS *topo.Server) { }) } if err != nil { - log.Fatalf("CreateTablet(%v): %v", tabletAlias, err) + return fmt.Errorf("CreateTablet(%v): %w", tabletAlias, err) } } } } + + return nil } // CopyShardReplications will create the ShardReplication objects in // the destination topo. -func CopyShardReplications(ctx context.Context, fromTS, toTS *topo.Server) { +func CopyShardReplications(ctx context.Context, fromTS, toTS *topo.Server) error { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { - log.Fatalf("fromTS.GetKeyspaces: %v", err) + return fmt.Errorf("fromTS.GetKeyspaces: %w", err) } cells, err := fromTS.GetCellInfoNames(ctx) if err != nil { - log.Fatalf("GetCellInfoNames(): %v", err) + return fmt.Errorf("GetCellInfoNames(): %w", err) } for _, keyspace := range keyspaces { shards, err := fromTS.GetShardNames(ctx, keyspace) if err != nil { - log.Fatalf("GetShardNames(%v): %v", keyspace, err) + return fmt.Errorf("GetShardNames(%v): %w", keyspace, err) } for _, shard := range shards { for _, cell := range cells { sri, err := fromTS.GetShardReplication(ctx, cell, keyspace, shard) if err != nil { - log.Fatalf("GetShardReplication(%v, %v, %v): %v", cell, keyspace, shard, err) + return fmt.Errorf("GetShardReplication(%v, %v, %v): %w", cell, keyspace, shard, err) } sriNodes := map[string]struct{}{} @@ -202,15 +208,19 @@ func CopyShardReplications(ctx context.Context, fromTS, toTS *topo.Server) { } } } + + return nil } // CopyRoutingRules will create the routing rules in the destination topo. -func CopyRoutingRules(ctx context.Context, fromTS, toTS *topo.Server) { +func CopyRoutingRules(ctx context.Context, fromTS, toTS *topo.Server) error { rr, err := fromTS.GetRoutingRules(ctx) if err != nil { - log.Fatalf("GetRoutingRules: %v", err) + return fmt.Errorf("GetRoutingRules: %w", err) } if err := toTS.SaveRoutingRules(ctx, rr); err != nil { log.Errorf("SaveRoutingRules(%v): %v", rr, err) } + + return nil }