Skip to content

Commit

Permalink
Merge branch 'main' into online-ddl-vrepl-suite-non-utf8
Browse files Browse the repository at this point in the history
Signed-off-by: Shlomi Noach <[email protected]>
  • Loading branch information
shlomi-noach committed Jun 8, 2021
2 parents 3fbc6bb + 50365bb commit 9e66dfe
Show file tree
Hide file tree
Showing 145 changed files with 3,151 additions and 15,943 deletions.
5 changes: 0 additions & 5 deletions docker/orchestrator/orchestrator.conf.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"AuditToSyslog": false,
"AuthenticationMethod": "",
"AuthUserHeader": "",
"AutoPseudoGTID": false,
"BackendDB": "sqlite",
"BinlogEventsChunkSize": 10000,
"CandidateInstanceExpireMinutes": 60,
Expand All @@ -19,7 +18,6 @@
"DetectInstanceAliasQuery": "SELECT value FROM _vt.local_metadata WHERE name='Alias'",
"DetectPromotionRuleQuery": "SELECT value FROM _vt.local_metadata WHERE name='PromotionRule'",
"DetectDataCenterQuery": "SELECT value FROM _vt.local_metadata WHERE name='DataCenter'",
"DetectPseudoGTIDQuery": "",
"DetectSemiSyncEnforcedQuery": "SELECT @@global.rpl_semi_sync_master_wait_no_slave AND @@global.rpl_semi_sync_master_timeout > 1000000",
"DiscoverByShowSlaveHosts": false,
"EnableSyslog": false,
Expand Down Expand Up @@ -77,8 +75,6 @@
],
"PromotionIgnoreHostnameFilters": [
],
"PseudoGTIDMonotonicHint": "asc:",
"PseudoGTIDPattern": "drop view if exists .*?`_pseudo_gtid_hint__",
"ReadLongRunningQueries": false,
"ReadOnly": false,
"ReasonableMaintenanceReplicationLagSeconds": 20,
Expand All @@ -97,7 +93,6 @@
"SkipBinlogEventsContaining": [
],
"SkipBinlogServerUnresolveCheck": true,
"SkipMaxScaleCheck": true,
"SkipOrchestratorDatabaseUpdate": false,
"SlaveStartPostWaitMilliseconds": 1000,
"SnapshotTopologiesIntervalHours": 0,
Expand Down
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,13 @@ require (
github.com/PuerkitoBio/goquery v1.5.1
github.com/aquarapid/vaultlib v0.5.1
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.28.8
github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13
github.com/cespare/xxhash/v2 v2.1.1
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
github.com/corpix/uarand v0.1.1 // indirect
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
github.com/dave/jennifer v1.4.1
github.com/evanphx/json-patch v4.5.0+incompatible
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab
Expand All @@ -40,7 +39,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/hashicorp/consul/api v1.5.0
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/serf v0.9.2 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps=
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM=
github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY=
github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
Expand Down
4 changes: 4 additions & 0 deletions go/cmd/vtadmin/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"vitess.io/vitess/go/vt/vtadmin/cluster"
"vitess.io/vitess/go/vt/vtadmin/grpcserver"
vtadminhttp "vitess.io/vitess/go/vt/vtadmin/http"
"vitess.io/vitess/go/vt/vtadmin/http/debug"
)

var (
Expand Down Expand Up @@ -129,6 +130,9 @@ func main() {
rootCmd.Flags().BoolVar(&httpOpts.EnableTracing, "http-tracing", false, "whether to enable tracing on the HTTP server")

rootCmd.Flags().BoolVar(&httpOpts.DisableCompression, "http-no-compress", false, "whether to disable compression of HTTP API responses")
rootCmd.Flags().BoolVar(&httpOpts.DisableDebug, "http-no-debug", false, "whether to disable /debug/pprof/* and /debug/env HTTP endpoints")
rootCmd.Flags().Var(&debug.OmitEnv, "http-debug-omit-env", "name of an environment variable to omit from /debug/env, if http debug endpoints are enabled. specify multiple times to omit multiple env vars")
rootCmd.Flags().Var(&debug.SanitizeEnv, "http-debug-sanitize-env", "name of an environment variable to sanitize in /debug/env, if http debug endpoints are enabled. specify multiple times to sanitize multiple env vars")
rootCmd.Flags().StringSliceVar(&httpOpts.CORSOrigins, "http-origin", []string{}, "repeated, comma-separated flag of allowed CORS origins. omit to disable CORS")
rootCmd.Flags().StringVar(&httpOpts.ExperimentalOptions.TabletURLTmpl,
"http-tablet-url-tmpl",
Expand Down
4 changes: 2 additions & 2 deletions go/cmd/vtbackup/vtbackup.go
Original file line number Diff line number Diff line change
Expand Up @@ -447,8 +447,8 @@ func startReplication(ctx context.Context, mysqld mysqlctl.MysqlDaemon, topoServ
}

// Stop replication (in case we're restarting), set master, and start replication.
if err := mysqld.SetMaster(ctx, ti.Tablet.MysqlHostname, int(ti.Tablet.MysqlPort), true /* stopReplicationBefore */, true /* startReplicationAfter */); err != nil {
return vterrors.Wrap(err, "MysqlDaemon.SetMaster failed")
if err := mysqld.SetReplicationSource(ctx, ti.Tablet.MysqlHostname, int(ti.Tablet.MysqlPort), true /* stopReplicationBefore */, true /* startReplicationAfter */); err != nil {
return vterrors.Wrap(err, "MysqlDaemon.SetReplicationSource failed")
}
return nil
}
Expand Down
93 changes: 93 additions & 0 deletions go/cmd/vtctldclient/internal/command/vschemas.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@ package command

import (
"fmt"
"io/ioutil"

"github.com/spf13/cobra"

"vitess.io/vitess/go/cmd/vtctldclient/cli"
"vitess.io/vitess/go/json2"

vschemapb "vitess.io/vitess/go/vt/proto/vschema"
vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
)

Expand All @@ -33,8 +36,89 @@ var (
Args: cobra.ExactArgs(1),
RunE: commandGetVSchema,
}
// ApplyVSchema makes an ApplyVSchema gRPC call to a vtctld.
ApplyVSchema = &cobra.Command{
Use: "ApplyVSchema {-vschema=<vschema> || -vschema-file=<vschema file> || -sql=<sql> || -sql-file=<sql file>} [-cells=c1,c2,...] [-skip-rebuild] [-dry-run] <keyspace>",
Args: cobra.ExactArgs(1),
DisableFlagsInUseLine: true,
RunE: commandApplyVSchema,
Short: "Applies the VTGate routing schema to the provided keyspace. Shows the result after application.",
}
)

var applyVSchemaOptions = struct {
VSchema string
VSchemaFile string
SQL string
SQLFile string
DryRun bool
SkipRebuild bool
Cells []string
}{}

func commandApplyVSchema(cmd *cobra.Command, args []string) error {
sqlMode := (applyVSchemaOptions.SQL != "") != (applyVSchemaOptions.SQLFile != "")
jsonMode := (applyVSchemaOptions.VSchema != "") != (applyVSchemaOptions.VSchemaFile != "")

if sqlMode && jsonMode {
return fmt.Errorf("only one of the sql, sql-file, vschema, or vschema-file flags may be specified when calling the ApplyVSchema command")
}

if !sqlMode && !jsonMode {
return fmt.Errorf("one of the sql, sql-file, vschema, or vschema-file flags must be specified when calling the ApplyVSchema command")
}

req := &vtctldatapb.ApplyVSchemaRequest{
Keyspace: cmd.Flags().Arg(0),
SkipRebuild: applyVSchemaOptions.SkipRebuild,
Cells: applyVSchemaOptions.Cells,
DryRun: applyVSchemaOptions.DryRun,
}

var err error
if sqlMode {
if applyVSchemaOptions.SQLFile != "" {
sqlBytes, err := ioutil.ReadFile(applyVSchemaOptions.SQLFile)
if err != nil {
return err
}
req.Sql = string(sqlBytes)
} else {
req.Sql = applyVSchemaOptions.SQL
}
} else { // jsonMode
var schema []byte
if applyVSchemaOptions.VSchemaFile != "" {
schema, err = ioutil.ReadFile(applyVSchemaOptions.VSchemaFile)
if err != nil {
return err
}
} else {
schema = []byte(applyVSchemaOptions.VSchema)
}

var vs *vschemapb.Keyspace
err = json2.Unmarshal(schema, vs)
if err != nil {
return err
}
req.VSchema = vs
}

cli.FinishedParsing(cmd)

res, err := client.ApplyVSchema(commandCtx, req)
if err != nil {
return err
}
data, err := cli.MarshalJSON(res.VSchema)
if err != nil {
return err
}
fmt.Printf("New VSchema object:\n%s\nIf this is not what you expected, check the input data (as JSON parsing will skip unexpected fields).\n", data)
return nil
}

func commandGetVSchema(cmd *cobra.Command, args []string) error {
cli.FinishedParsing(cmd)

Expand All @@ -58,5 +142,14 @@ func commandGetVSchema(cmd *cobra.Command, args []string) error {
}

func init() {
ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.VSchema, "vschema", "", "VSchema")
ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.VSchemaFile, "vschema-file", "", "VSchema File")
ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.SQL, "sql", "", "A VSchema DDL SQL statement, e.g. `alter table t add vindex hash(id)`")
ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.SQLFile, "sql-file", "", "A file containing VSchema DDL SQL")
ApplyVSchema.Flags().BoolVar(&applyVSchemaOptions.DryRun, "dry-run", false, "If set, do not save the altered vschema, simply echo to console.")
ApplyVSchema.Flags().BoolVar(&applyVSchemaOptions.SkipRebuild, "skip-rebuild", false, "If set, do no rebuild the SrvSchema objects.")
ApplyVSchema.Flags().StringSliceVar(&applyVSchemaOptions.Cells, "cells", nil, "If specified, limits the rebuild to the cells, after upload. Ignored if skipRebuild is set.")
Root.AddCommand(ApplyVSchema)

Root.AddCommand(GetVSchema)
}
1 change: 0 additions & 1 deletion go/cmd/vtorc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ func main() {
verbose := flag.Bool("verbose", false, "verbose")
debug := flag.Bool("debug", false, "debug mode (very verbose)")
stack := flag.Bool("stack", false, "add stack trace upon error")
config.RuntimeCLIFlags.SkipBinlogSearch = flag.Bool("skip-binlog-search", false, "when matching via Pseudo-GTID, only use relay logs. This can save the hassle of searching for a non-existend pseudo-GTID entry, for example in servers with replication filters.")
config.RuntimeCLIFlags.SkipUnresolve = flag.Bool("skip-unresolve", false, "Do not unresolve a host name")
config.RuntimeCLIFlags.SkipUnresolveCheck = flag.Bool("skip-unresolve-check", false, "Skip/ignore checking an unresolve mapping (via hostname_unresolve table) resolves back to same hostname")
config.RuntimeCLIFlags.Noop = flag.Bool("noop", false, "Dry run; do not perform destructing operations")
Expand Down
80 changes: 80 additions & 0 deletions go/flagutil/sets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
Copyright 2021 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flagutil

import (
"flag"
"strings"

"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/sets"
)

var (
_ flag.Value = (*StringSetFlag)(nil)
_ pflag.Value = (*StringSetFlag)(nil)
)

// StringSetFlag can be used to collect multiple instances of a flag into a set
// of values.
//
// For example, defining the following:
//
// var x flagutil.StringSetFlag
// flag.Var(&x, "foo", "")
//
// And then specifying "-foo x -foo y -foo x", will result in a set of {x, y}.
//
// In addition to implemnting the standard flag.Value interface, it also
// provides an implementation of pflag.Value, so it is usable in libraries like
// cobra.
type StringSetFlag struct {
set sets.String
}

// ToSet returns the underlying string set, or an empty set if the underlying
// set is nil.
func (set *StringSetFlag) ToSet() sets.String {
if set.set == nil {
set.set = sets.NewString()
}

return set.set
}

// Set is part of the pflag.Value and flag.Value interfaces.
func (set *StringSetFlag) Set(s string) error {
if set.set == nil {
set.set = sets.NewString(s)
return nil
}

set.set.Insert(s)
return nil
}

// String is part of the pflag.Value and flag.Value interfaces.
func (set *StringSetFlag) String() string {
if set.set == nil {
return ""
}

return strings.Join(set.set.List(), ", ")
}

// Type is part of the pflag.Value interface.
func (set *StringSetFlag) Type() string { return "StringSetFlag" }
Loading

0 comments on commit 9e66dfe

Please sign in to comment.