diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 9b4b9cebca9..ce408309077 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -45,17 +45,14 @@ import ( "vitess.io/vitess/go/vt/vttablet/tmclient" querypb "vitess.io/vitess/go/vt/proto/query" - "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" "vitess.io/vitess/go/vt/proto/vttime" ) -func init() { - *tmclient.TabletManagerProtocol = testutil.TabletManagerClientProtocol -} - func TestGetClusters(t *testing.T) { tests := []struct { name string @@ -277,11 +274,13 @@ func TestGetKeyspaces(t *testing.T) { } func TestGetSchemas(t *testing.T) { + t.Parallel() + tests := []struct { name string clusterTablets [][]*vtadminpb.Tablet // Indexed by tablet alias - tabletSchemas map[string]*tabletmanagerdata.SchemaDefinition + tabletSchemas map[string]*tabletmanagerdatapb.SchemaDefinition req *vtadminpb.GetSchemasRequest expected *vtadminpb.GetSchemasResponse }{ @@ -315,10 +314,10 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{ + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ "c0_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -345,7 +344,7 @@ func TestGetSchemas(t *testing.T) { Name: "cluster0", }, Keyspace: "commerce", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -395,10 +394,10 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{ + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ "c0_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -417,7 +416,7 @@ func TestGetSchemas(t *testing.T) { }, "c1_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t2", Schema: `CREATE TABLE t2 (id int(11) not null,PRIMARY KEY (id));`, @@ -444,7 +443,7 @@ func TestGetSchemas(t *testing.T) { Name: "cluster0", }, Keyspace: "commerce", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -467,7 +466,7 @@ func TestGetSchemas(t *testing.T) { Name: "cluster1", }, Keyspace: "commerce", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t2", Schema: `CREATE TABLE t2 (id int(11) not null,PRIMARY KEY (id));`, @@ -517,10 +516,10 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{ + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ "c0_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -539,7 +538,7 @@ func TestGetSchemas(t *testing.T) { }, "c1_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t2", Schema: `CREATE TABLE t2 (id int(11) not null,PRIMARY KEY (id));`, @@ -568,7 +567,7 @@ func TestGetSchemas(t *testing.T) { Name: "cluster1", }, Keyspace: "commerce", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t2", Schema: `CREATE TABLE t2 (id int(11) not null,PRIMARY KEY (id));`, @@ -605,10 +604,10 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{ + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ "c0_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -650,7 +649,7 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{}, + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{}, req: &vtadminpb.GetSchemasRequest{}, expected: &vtadminpb.GetSchemasResponse{ Schemas: []*vtadminpb.Schema{}, @@ -673,10 +672,10 @@ func TestGetSchemas(t *testing.T) { }, }, }, - tabletSchemas: map[string]*tabletmanagerdata.SchemaDefinition{ + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ "c0_cell1-0000000100": { DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "t1", Schema: `CREATE TABLE t1 (id int(11) not null,PRIMARY KEY (id));`, @@ -702,50 +701,76 @@ func TestGetSchemas(t *testing.T) { } for _, tt := range tests { - testutil.TabletManagerClient.Schemas = map[string]*tabletmanagerdata.SchemaDefinition{} - - topos := []*topo.Server{ - memorytopo.NewServer("c0_cell1"), - memorytopo.NewServer("c1_cell1"), - } - - // Setting up WithTestServer in a generic, recursive way is... unpleasant, - // so all tests are set-up and run in the context of these two clusters. - testutil.WithTestServer(t, grpcvtctldserver.NewVtctldServer(topos[0]), func(t *testing.T, cluster0Client vtctldclient.VtctldClient) { - testutil.WithTestServer(t, grpcvtctldserver.NewVtctldServer(topos[1]), func(t *testing.T, cluster1Client vtctldclient.VtctldClient) { - // Put 'em in a slice so we can look them up by index - clusterClients := []vtctldclient.VtctldClient{cluster0Client, cluster1Client} - - // Build the clusters - clusters := make([]*cluster.Cluster, len(topos)) - for cdx, toposerver := range topos { - // Handle when a test doesn't define any tablets for a given cluster. - var cts []*vtadminpb.Tablet - if cdx < len(tt.clusterTablets) { - cts = tt.clusterTablets[cdx] - } + tt := tt - for _, tablet := range cts { - // AddTablet also adds the keyspace + shard for us. - testutil.AddTablet(context.Background(), t, toposerver, tablet.Tablet, nil) + t.Run(tt.name, func(t *testing.T) { + t.Parallel() - // Adds each SchemaDefinition to the fake TabletManagerClient, or nil - // if there are no schemas for that tablet. (All tablet aliases must - // exist in the map. Otherwise, TabletManagerClient will return an error when - // looking up the schema with tablet alias that doesn't exist.) - alias := topoproto.TabletAliasString(tablet.Tablet.Alias) - testutil.TabletManagerClient.Schemas[alias] = tt.tabletSchemas[alias] - } + topos := []*topo.Server{ + memorytopo.NewServer("c0_cell1"), + memorytopo.NewServer("c1_cell1"), + } - clusters[cdx] = buildCluster(cdx, clusterClients[cdx], cts, nil) - } + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{}, + } + + vtctlds := []vtctlservicepb.VtctldServer{ + testutil.NewVtctldServerWithTabletManagerClient(t, topos[0], &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return grpcvtctldserver.NewVtctldServer(ts) + }), + testutil.NewVtctldServerWithTabletManagerClient(t, topos[1], &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return grpcvtctldserver.NewVtctldServer(ts) + }), + } - api := NewAPI(clusters, grpcserver.Options{}, http.Options{}) + // Setting up WithTestServer in a generic, recursive way is... unpleasant, + // so all tests are set-up and run in the context of these two clusters. + testutil.WithTestServer(t, vtctlds[0], func(t *testing.T, cluster0Client vtctldclient.VtctldClient) { + testutil.WithTestServer(t, vtctlds[1], func(t *testing.T, cluster1Client vtctldclient.VtctldClient) { + // Put 'em in a slice so we can look them up by index + clusterClients := []vtctldclient.VtctldClient{cluster0Client, cluster1Client} - resp, err := api.GetSchemas(context.Background(), tt.req) - require.NoError(t, err) + // Build the clusters + clusters := make([]*cluster.Cluster, len(topos)) + for cdx, toposerver := range topos { + // Handle when a test doesn't define any tablets for a given cluster. + var cts []*vtadminpb.Tablet + if cdx < len(tt.clusterTablets) { + cts = tt.clusterTablets[cdx] + } - vtadmintestutil.AssertSchemaSlicesEqual(t, tt.expected.Schemas, resp.Schemas, tt.name) + for _, tablet := range cts { + // AddTablet also adds the keyspace + shard for us. + testutil.AddTablet(context.Background(), t, toposerver, tablet.Tablet, nil) + + // Adds each SchemaDefinition to the fake TabletManagerClient, or nil + // if there are no schemas for that tablet. (All tablet aliases must + // exist in the map. Otherwise, TabletManagerClient will return an error when + // looking up the schema with tablet alias that doesn't exist.) + alias := topoproto.TabletAliasString(tablet.Tablet.Alias) + tmc.GetSchemaResults[alias] = struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + Schema: tt.tabletSchemas[alias], + Error: nil, + } + } + + clusters[cdx] = buildCluster(cdx, clusterClients[cdx], cts, nil) + } + + api := NewAPI(clusters, grpcserver.Options{}, http.Options{}) + + resp, err := api.GetSchemas(context.Background(), tt.req) + require.NoError(t, err) + + vtadmintestutil.AssertSchemaSlicesEqual(t, tt.expected.Schemas, resp.Schemas, tt.name) + }) }) }) } @@ -1234,3 +1259,17 @@ func buildCluster(i int, vtctldClient vtctldclient.VtctldClient, tablets []*vtad return cluster } + +func init() { + // For tests that don't actually care about mocking the tmclient (i.e. they + // call grpcvtctldserver.NewVtctldServer to initialize the unit under test), + // this needs to be set. + // + // Tests that do care about the tmclient should use + // testutil.NewVtctldServerWithTabletManagerClient to initialize their + // VtctldServer. + *tmclient.TabletManagerProtocol = "vtadmin.test" + tmclient.RegisterTabletManagerClientFactory("vtadmin.test", func() tmclient.TabletManagerClient { + return nil + }) +} diff --git a/go/vt/vtctl/endtoend/get_schema_test.go b/go/vt/vtctl/endtoend/get_schema_test.go index 1e93ce633c3..1fb10fff0f0 100644 --- a/go/vt/vtctl/endtoend/get_schema_test.go +++ b/go/vt/vtctl/endtoend/get_schema_test.go @@ -138,14 +138,26 @@ func TestGetSchema(t *testing.T) { }, } - tmc := testutil.TabletManagerClient - tmc.Schemas[topoproto.TabletAliasString(tablet.Alias)] = sd + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + topoproto.TabletAliasString(tablet.Alias): { + Schema: sd, + Error: nil, + }, + }, + } - *tmclient.TabletManagerProtocol = testutil.TabletManagerClientProtocol + tmclient.RegisterTabletManagerClientFactory(t.Name(), func() tmclient.TabletManagerClient { + return &tmc + }) + *tmclient.TabletManagerProtocol = t.Name() logger := logutil.NewMemoryLogger() - err := vtctl.RunCommand(ctx, wrangler.New(logger, topo, tmc), []string{ + err := vtctl.RunCommand(ctx, wrangler.New(logger, topo, &tmc), []string{ "GetSchema", topoproto.TabletAliasString(tablet.Alias), }) @@ -185,7 +197,7 @@ func TestGetSchema(t *testing.T) { }, } - err = vtctl.RunCommand(ctx, wrangler.New(logger, topo, tmc), []string{ + err = vtctl.RunCommand(ctx, wrangler.New(logger, topo, &tmc), []string{ "GetSchema", "-table_sizes_only", topoproto.TabletAliasString(tablet.Alias), @@ -202,12 +214,3 @@ func TestGetSchema(t *testing.T) { assert.Equal(t, sd, actual) } - -func init() { - // enforce we will use the right protocol (gRPC) (note the - // client is unused, but it is initialized, so it needs to exist) - *tmclient.TabletManagerProtocol = "grpc" - tmclient.RegisterTabletManagerClientFactory("grpc", func() tmclient.TabletManagerClient { - return nil - }) -} diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 9861c14ee30..db860a983cc 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -40,15 +40,29 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" "vitess.io/vitess/go/vt/proto/vttime" ) func init() { - *tmclient.TabletManagerProtocol = testutil.TabletManagerClientProtocol *backupstorage.BackupStorageImplementation = testutil.BackupStorageImplementation + + // For tests that don't actually care about mocking the tmclient (i.e. they + // call NewVtctldServer to initialize the unit under test), this needs to be + // set. + // + // Tests that do care about the tmclient should use + // testutil.NewVtctldServerWithTabletManagerClient to initialize their + // VtctldServer. + *tmclient.TabletManagerProtocol = "grpcvtctldserver.test" + tmclient.RegisterTabletManagerClientFactory("grpcvtctldserver.test", func() tmclient.TabletManagerClient { + return nil + }) } func TestChangeTabletType(t *testing.T) { + t.Parallel() + tests := []struct { name string cells []string @@ -203,11 +217,16 @@ func TestChangeTabletType(t *testing.T) { } for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() ts := memorytopo.NewServer(tt.cells...) - vtctld := NewVtctldServer(ts) - testutil.TabletManagerClient.Topo = ts + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, &testutil.TabletManagerClient{ + TopoServer: ts, + }, func(ts *topo.Server) vtctlservicepb.VtctldServer { return NewVtctldServer(ts) }) testutil.AddTablets(ctx, t, ts, nil, tt.tablets...) @@ -244,10 +263,13 @@ func TestChangeTabletType(t *testing.T) { } t.Run("tabletmanager failure", func(t *testing.T) { + t.Parallel() + ctx := context.Background() ts := memorytopo.NewServer("zone1") - vtctld := NewVtctldServer(ts) - testutil.TabletManagerClient.Topo = nil + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, &testutil.TabletManagerClient{ + TopoServer: nil, + }, func(ts *topo.Server) vtctlservicepb.VtctldServer { return NewVtctldServer(ts) }) testutil.AddTablet(ctx, t, ts, &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -2186,7 +2208,15 @@ func TestGetTablet(t *testing.T) { func TestGetSchema(t *testing.T) { ctx := context.Background() ts := memorytopo.NewServer("zone1") - vtctld := NewVtctldServer(ts) + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{}, + } + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return NewVtctldServer(ts) + }) validAlias := &topodatapb.TabletAlias{ Cell: "zone1", @@ -2205,27 +2235,33 @@ func TestGetSchema(t *testing.T) { // we need to run this on each test case or they will pollute each other setupSchema := func() { - testutil.TabletManagerClient.Schemas[topoproto.TabletAliasString(validAlias)] = &tabletmanagerdatapb.SchemaDefinition{ - DatabaseSchema: "CREATE DATABASE vt_testkeyspace", - TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ - { - Name: "t1", - Schema: `CREATE TABLE t1 ( + tmc.GetSchemaResults[topoproto.TabletAliasString(validAlias)] = struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + Schema: &tabletmanagerdatapb.SchemaDefinition{ + DatabaseSchema: "CREATE DATABASE vt_testkeyspace", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: `CREATE TABLE t1 ( id int(11) not null, PRIMARY KEY (id) );`, - Type: "BASE", - Columns: []string{"id"}, - DataLength: 100, - RowCount: 50, - Fields: []*querypb.Field{ - { - Name: "id", - Type: querypb.Type_INT32, + Type: "BASE", + Columns: []string{"id"}, + DataLength: 100, + RowCount: 50, + Fields: []*querypb.Field{ + { + Name: "id", + Type: querypb.Type_INT32, + }, }, }, }, }, + Error: nil, } } diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 414f794d694..0fd14bd75ae 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -19,6 +19,9 @@ package testutil import ( "context" "fmt" + "sync" + "testing" + "time" "github.com/stretchr/testify/assert" @@ -27,54 +30,502 @@ import ( "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vttablet/tmclient" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" "vitess.io/vitess/go/vt/proto/vttime" ) -// tabletManagerClient implements the tmclient.TabletManagerClient for -// testing. It allows users to mock various tmclient methods. -type tabletManagerClient struct { +var ( + tmclientLock sync.Mutex + tmclientFactoryLock sync.Mutex + tmclients = map[string]tmclient.TabletManagerClient{} + tmclientFactories = map[string]func() tmclient.TabletManagerClient{} +) + +// NewVtctldServerWithTabletManagerClient returns a new +// grpcvtctldserver.VtctldServer configured with the given topo server and +// tmclient.TabletManagerClient implementation for testing. +// +// It synchronizes on private locks to prevent multiple goroutines from stepping +// on each other during VtctldServer initialization, but still run the rest of +// the test in parallel. +// +// NOTE, THE FIRST: It is only safe to use in parallel with other tests using +// this method of creating a VtctldServer, or with tests that do not depend on a +// VtctldServer's tmclient.TabletManagerClient implementation. +// +// NOTE, THE SECOND: It needs to register a unique name to the tmclient factory +// registry, so we keep a shadow map of factories registered for "protocols" by +// this function. That way, if we happen to have multiple tests with the same +// name, we can swap out the return value for the factory and allow both tests +// to run, rather than the second test failing when it attempts to register a +// second factory for the same "protocol" name. +// +// NOTE, THE THIRD: we take a "new" func to produce a valid +// vtctlservicepb.VtctldServer implementation, rather than constructing directly +// ourselves with grpcvtctldserver.NewVtctldServer. This is to prevent an import +// cycle between this package and package grpcvtctldserver. Further, because the +// return type of NewVtctldServer is the struct type +// (*grpcvtctldserver.VtctldServer) and not the interface type +// vtctlservicepb.VtctldServer, tests will need to indirect that call through an +// extra layer rather than passing the function identifier directly, e.g.: +// +// vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, &testutil.TabletManagerClient{ +// ... +// }, func(ts *topo.Server) vtctlservicepb.VtctldServer { return NewVtctldServer(ts) }) +// +func NewVtctldServerWithTabletManagerClient(t *testing.T, ts *topo.Server, tmc tmclient.TabletManagerClient, newVtctldServerFn func(ts *topo.Server) vtctlservicepb.VtctldServer) vtctlservicepb.VtctldServer { + tmclientFactoryLock.Lock() + defer tmclientFactoryLock.Unlock() + + protocol := t.Name() + + if _, alreadyRegisteredFactory := tmclientFactories[protocol]; !alreadyRegisteredFactory { + factory := func() tmclient.TabletManagerClient { + tmclientLock.Lock() + defer tmclientLock.Unlock() + + client, ok := tmclients[protocol] + if !ok { + t.Fatal("Test managed to register a factory for a client value that never got set; this should be impossible") + } + + return client + } + + tmclient.RegisterTabletManagerClientFactory(protocol, factory) + tmclientFactories[protocol] = factory + } + + // Always swap in the new client return value for the given protocol name. + // We cannot defer the unlock here, because grpcvtctldserver.NewVtctldServer + // eventually will call into the factory we registered above, and we will + // deadlock ourselves. + tmclientLock.Lock() + tmclients[protocol] = tmc + tmclientLock.Unlock() + + // Be (mostly, we can't help concurrent goroutines not using this function) + // atomic with our mutation of the global TabletManagerProtocol pointer. + oldProto := *tmclient.TabletManagerProtocol + defer func() { *tmclient.TabletManagerProtocol = oldProto }() + + *tmclient.TabletManagerProtocol = protocol + + return newVtctldServerFn(ts) +} + +// TabletManagerClient implements the tmclient.TabletManagerClient interface +// with mock delays and response values, for use in unit tests. +type TabletManagerClient struct { tmclient.TabletManagerClient - Topo *topo.Server - Schemas map[string]*tabletmanagerdatapb.SchemaDefinition + // TopoServer is used for certain TabletManagerClient rpcs that update topo + // information, e.g. ChangeType. To force an error result for those rpcs in + // a test, set tmc.TopoServer = nil. + TopoServer *topo.Server + // keyed by tablet alias. + DemoteMasterDelays map[string]time.Duration + // keyed by tablet alias. + DemoteMasterResults map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + } + // keyed by tablet alias. + GetSchemaDelays map[string]time.Duration + // keyed by tablet alias. + GetSchemaResults map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + } + // keyed by tablet alias. + MasterPositionDelays map[string]time.Duration + // keyed by tablet alias. + MasterPositionResults map[string]struct { + Position string + Error error + } + // keyed by tablet alias. + PopulateReparentJournalDelays map[string]time.Duration + // keyed by tablet alias + PopulateReparentJournalResults map[string]error + // keyed by tablet alias. + PromoteReplicaDelays map[string]time.Duration + // keyed by tablet alias. injects a sleep to the end of the function + // regardless of parent context timeout or error result. + PromoteReplicaPostDelays map[string]time.Duration + // keyed by tablet alias. + PromoteReplicaResults map[string]struct { + Result string + Error error + } + ReplicationStatusResults map[string]struct { + Position *replicationdatapb.Status + Error error + } + // keyed by tablet alias. + SetMasterDelays map[string]time.Duration + // keyed by tablet alias. + SetMasterResults map[string]error + // keyed by tablet alias. + SetReadWriteDelays map[string]time.Duration + // keyed by tablet alias. + SetReadWriteResults map[string]error + // keyed by tablet alias. + StopReplicationAndGetStatusDelays map[string]time.Duration + // keyed by tablet alias. + StopReplicationAndGetStatusResults map[string]struct { + Status *replicationdatapb.Status + StopStatus *replicationdatapb.StopReplicationStatus + Error error + } + // keyed by tablet alias. + WaitForPositionDelays map[string]time.Duration + // keyed by tablet alias. injects a sleep to the end of the function + // regardless of parent context timeout or error result. + WaitForPositionPostDelays map[string]time.Duration + // WaitForPosition(tablet *topodatapb.Tablet, position string) error, so we + // key by tablet alias and then by position. + WaitForPositionResults map[string]map[string]error + // keyed by tablet alias. + UndoDemoteMasterDelays map[string]time.Duration + // keyed by tablet alias + UndoDemoteMasterResults map[string]error } // ChangeType is part of the tmclient.TabletManagerClient interface. -func (c *tabletManagerClient) ChangeType(ctx context.Context, tablet *topodatapb.Tablet, newType topodatapb.TabletType) error { - if c.Topo == nil { +func (fake *TabletManagerClient) ChangeType(ctx context.Context, tablet *topodatapb.Tablet, newType topodatapb.TabletType) error { + if fake.TopoServer == nil { return assert.AnError } - _, err := topotools.ChangeType(ctx, c.Topo, tablet.Alias, newType, &vttime.Time{}) + _, err := topotools.ChangeType(ctx, fake.TopoServer, tablet.Alias, newType, &vttime.Time{}) return err } +// DemoteMaster is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) DemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.MasterStatus, error) { + if fake.DemoteMasterResults == nil { + return nil, assert.AnError + } + + if tablet.Alias == nil { + return nil, assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.DemoteMasterDelays != nil { + if delay, ok := fake.DemoteMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.DemoteMasterResults[key]; ok { + return result.Status, result.Error + } + + return nil, assert.AnError +} + // GetSchema is part of the tmclient.TabletManagerClient interface. -func (c *tabletManagerClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, tablets []string, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { +func (fake *TabletManagerClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, tablets []string, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { + if fake.GetSchemaResults == nil { + return nil, assert.AnError + } + + if tablet.Alias == nil { + return nil, assert.AnError + } + key := topoproto.TabletAliasString(tablet.Alias) - schema, ok := c.Schemas[key] - if !ok { - return nil, fmt.Errorf("no schemas for %s", key) + if fake.GetSchemaDelays != nil { + if delay, ok := fake.GetSchemaDelays[key]; ok { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + // proceed to results + } + } } - return schema, nil + if result, ok := fake.GetSchemaResults[key]; ok { + return result.Schema, result.Error + } + + return nil, fmt.Errorf("%w: no schemas for %s", assert.AnError, key) +} + +// MasterPosition is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) MasterPosition(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + if fake.MasterPositionResults == nil { + return "", assert.AnError + } + + if tablet.Alias == nil { + return "", assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.MasterPositionDelays != nil { + if delay, ok := fake.MasterPositionDelays[key]; ok { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.MasterPositionResults[key]; ok { + return result.Position, result.Error + } + + return "", assert.AnError +} + +// PopulateReparentJournal is part of the tmclient.TabletManagerClient +// interface. +func (fake *TabletManagerClient) PopulateReparentJournal(ctx context.Context, tablet *topodatapb.Tablet, timeCreatedNS int64, actionName string, primaryAlias *topodatapb.TabletAlias, pos string) error { + if fake.PopulateReparentJournalResults == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.PopulateReparentJournalDelays != nil { + if delay, ok := fake.PopulateReparentJournalDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + if result, ok := fake.PopulateReparentJournalResults[key]; ok { + return result + } + + return assert.AnError } -// TabletManagerClientProtocol is the protocol this package registers its client -// test implementation under. Users should set *tmclient.TabletManagerProtocol -// to this value before use. -const TabletManagerClientProtocol = "grpcvtctldserver.testutil" +// PromoteReplica is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + if fake.PromoteReplicaResults == nil { + return "", assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + defer func() { + if fake.PromoteReplicaPostDelays == nil { + return + } + + if delay, ok := fake.PromoteReplicaPostDelays[key]; ok { + time.Sleep(delay) + } + }() -// TabletManagerClient is the singleton test client instance. It is public and -// singleton to allow tests to mutate and verify its state. -var TabletManagerClient = &tabletManagerClient{ - Schemas: map[string]*tabletmanagerdatapb.SchemaDefinition{}, + if fake.PromoteReplicaDelays != nil { + if delay, ok := fake.PromoteReplicaDelays[key]; ok { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.PromoteReplicaResults[key]; ok { + return result.Result, result.Error + } + + return "", assert.AnError } -func init() { - tmclient.RegisterTabletManagerClientFactory(TabletManagerClientProtocol, func() tmclient.TabletManagerClient { - return TabletManagerClient - }) +// ReplicationStatus is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) ReplicationStatus(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.Status, error) { + if fake.ReplicationStatusResults == nil { + return nil, assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if result, ok := fake.ReplicationStatusResults[key]; ok { + return result.Position, result.Error + } + + return nil, assert.AnError +} + +// SetMaster is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { + if fake.SetMasterResults == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.SetMasterDelays != nil { + if delay, ok := fake.SetMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.SetMasterResults[key]; ok { + return result + } + + return assert.AnError +} + +// SetReadWrite is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) SetReadWrite(ctx context.Context, tablet *topodatapb.Tablet) error { + if fake.SetReadWriteResults == nil { + return assert.AnError + } + + if tablet.Alias == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.SetReadWriteDelays != nil { + if delay, ok := fake.SetReadWriteDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if err, ok := fake.SetReadWriteResults[key]; ok { + return err + } + + return assert.AnError +} + +// StopReplicationAndGetStatus is part of the tmclient.TabletManagerClient +// interface. +func (fake *TabletManagerClient) StopReplicationAndGetStatus(ctx context.Context, tablet *topodatapb.Tablet, mode replicationdatapb.StopReplicationMode) (*replicationdatapb.Status, *replicationdatapb.StopReplicationStatus, error) { + if fake.StopReplicationAndGetStatusResults == nil { + return nil, nil, assert.AnError + } + + if tablet.Alias == nil { + return nil, nil, assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.StopReplicationAndGetStatusDelays != nil { + if delay, ok := fake.StopReplicationAndGetStatusDelays[key]; ok { + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.StopReplicationAndGetStatusResults[key]; ok { + return result.Status, result.StopStatus, result.Error + } + + return nil, nil, assert.AnError +} + +// WaitForPosition is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) WaitForPosition(ctx context.Context, tablet *topodatapb.Tablet, position string) error { + tabletKey := topoproto.TabletAliasString(tablet.Alias) + + defer func() { + if fake.WaitForPositionPostDelays == nil { + return + } + + if delay, ok := fake.WaitForPositionPostDelays[tabletKey]; ok { + time.Sleep(delay) + } + }() + + if fake.WaitForPositionDelays != nil { + if delay, ok := fake.WaitForPositionDelays[tabletKey]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if fake.WaitForPositionResults == nil { + return assert.AnError + } + + tabletResultsByPosition, ok := fake.WaitForPositionResults[tabletKey] + if !ok { + return assert.AnError + } + + result, ok := tabletResultsByPosition[position] + if !ok { + return assert.AnError + } + + return result +} + +// UndoDemoteMaster is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) UndoDemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) error { + if fake.UndoDemoteMasterResults == nil { + return assert.AnError + } + + if tablet.Alias == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.UndoDemoteMasterDelays != nil { + if delay, ok := fake.UndoDemoteMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.UndoDemoteMasterResults[key]; ok { + return result + } + + return assert.AnError } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 95ba0b9e885..56e0dba1492 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -29,10 +29,8 @@ import ( "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" - "vitess.io/vitess/go/vt/vttablet/tmclient" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -115,7 +113,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { name string // setup ts *topo.Server - tmc *emergencyReparenterTestTMClient + tmc *testutil.TabletManagerClient unlockTopo bool shards []*vtctldatapb.Shard tablets []*topodatapb.Tablet @@ -129,7 +127,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "success", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, }, @@ -234,7 +232,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { // zone1-101. name: "success with requested primary-elect", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, }, @@ -341,7 +339,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "shard not found", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{}, + tmc: &testutil.TabletManagerClient{}, unlockTopo: true, // we shouldn't try to lock the nonexistent shard shards: nil, keyspace: "testkeyspace", @@ -352,7 +350,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "cannot stop replication", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -410,7 +408,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "lost topo lock", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -468,7 +466,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "cannot get reparent candidates", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -538,7 +536,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "zero valid reparent candidates", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{}, + tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { Keyspace: "testkeyspace", @@ -553,7 +551,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "error waiting for relay logs to apply", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -643,7 +641,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "requested primary-elect is not in tablet map", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -731,7 +729,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "requested primary-elect is not winning primary-elect", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -820,7 +818,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "cannot promote new primary", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string Error error @@ -965,7 +963,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { tests := []struct { name string ts *topo.Server - tmc *emergencyReparenterTestTMClient + tmc *testutil.TabletManagerClient unlockTopo bool keyspace string shard string @@ -978,7 +976,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "success", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -1058,7 +1056,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "primary not in tablet map", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{}, + tmc: &testutil.TabletManagerClient{}, keyspace: "testkeyspace", shard: "-", newPrimaryTabletAlias: "zone2-0000000200", @@ -1073,7 +1071,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "PromoteReplica error", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string Error error @@ -1111,7 +1109,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "lost topology lock", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string Error error @@ -1150,7 +1148,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "cannot repopulate reparent journal on new primary", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, }, @@ -1191,7 +1189,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "all replicas failing to SetMaster does fail the promotion", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -1245,7 +1243,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "all replicas slow to SetMaster does fail the promotion", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -1305,7 +1303,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { { name: "one replica failing to SetMaster does not fail the promotion", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -1410,7 +1408,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { } tests := []struct { name string - tmc *emergencyReparenterTestTMClient + tmc *testutil.TabletManagerClient candidates map[string]mysql.Position tabletMap map[string]*topo.TabletInfo statusMap map[string]*replicationdatapb.StopReplicationStatus @@ -1418,7 +1416,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }{ { name: "all tablet pass", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ WaitForPositionResults: map[string]map[string]error{ "zone1-0000000100": { "position1": nil, @@ -1466,7 +1464,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }, { name: "one tablet fails", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ WaitForPositionResults: map[string]map[string]error{ "zone1-0000000100": { "position1": nil, @@ -1514,7 +1512,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }, { name: "multiple tablets fail", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ WaitForPositionResults: map[string]map[string]error{ "zone1-0000000100": { "position1": nil, @@ -1579,7 +1577,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }, { name: "one slow tablet", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ WaitForPositionDelays: map[string]time.Duration{ "zone1-0000000101": time.Minute, }, @@ -1647,351 +1645,3 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }) } } - -type emergencyReparenterTestTMClient struct { - tmclient.TabletManagerClient - // keyed by tablet alias. - DemoteMasterDelays map[string]time.Duration - // keyed by tablet alias. - DemoteMasterResults map[string]struct { - Status *replicationdatapb.MasterStatus - Error error - } - // keyed by tablet alias. - MasterPositionDelays map[string]time.Duration - // keyed by tablet alias. - MasterPositionResults map[string]struct { - Position string - Error error - } - // keyed by tablet alias. - PopulateReparentJournalDelays map[string]time.Duration - // keyed by tablet alias - PopulateReparentJournalResults map[string]error - // keyed by tablet alias. - PromoteReplicaDelays map[string]time.Duration - // keyed by tablet alias. injects a sleep to the end of the function - // regardless of parent context timeout or error result. - PromoteReplicaPostDelays map[string]time.Duration - // keyed by tablet alias. - PromoteReplicaResults map[string]struct { - Result string - Error error - } - ReplicationStatusResults map[string]struct { - Position *replicationdatapb.Status - Error error - } - // keyed by tablet alias. - SetMasterDelays map[string]time.Duration - // keyed by tablet alias. - SetMasterResults map[string]error - // keyed by tablet alias. - SetReadWriteDelays map[string]time.Duration - // keyed by tablet alias. - SetReadWriteResults map[string]error - // keyed by tablet alias. - StopReplicationAndGetStatusDelays map[string]time.Duration - // keyed by tablet alias. - StopReplicationAndGetStatusResults map[string]struct { - Status *replicationdatapb.Status - StopStatus *replicationdatapb.StopReplicationStatus - Error error - } - // keyed by tablet alias. - WaitForPositionDelays map[string]time.Duration - // keyed by tablet alias. injects a sleep to the end of the function - // regardless of parent context timeout or error result. - WaitForPositionPostDelays map[string]time.Duration - // WaitForPosition(tablet *topodatapb.Tablet, position string) error, so we - // key by tablet alias and then by position. - WaitForPositionResults map[string]map[string]error - // keyed by tablet alias. - UndoDemoteMasterDelays map[string]time.Duration - // keyed by tablet alias - UndoDemoteMasterResults map[string]error -} - -func (fake *emergencyReparenterTestTMClient) DemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.MasterStatus, error) { - if fake.DemoteMasterResults == nil { - return nil, assert.AnError - } - - if tablet.Alias == nil { - return nil, assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.DemoteMasterDelays != nil { - if delay, ok := fake.DemoteMasterDelays[key]; ok { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.DemoteMasterResults[key]; ok { - return result.Status, result.Error - } - - return nil, assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) MasterPosition(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { - if fake.MasterPositionResults == nil { - return "", assert.AnError - } - - if tablet.Alias == nil { - return "", assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.MasterPositionDelays != nil { - if delay, ok := fake.MasterPositionDelays[key]; ok { - select { - case <-ctx.Done(): - return "", ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.MasterPositionResults[key]; ok { - return result.Position, result.Error - } - - return "", assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) PopulateReparentJournal(ctx context.Context, tablet *topodatapb.Tablet, timeCreatedNS int64, actionName string, primaryAlias *topodatapb.TabletAlias, pos string) error { - if fake.PopulateReparentJournalResults == nil { - return assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.PopulateReparentJournalDelays != nil { - if delay, ok := fake.PopulateReparentJournalDelays[key]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - if result, ok := fake.PopulateReparentJournalResults[key]; ok { - return result - } - - return assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { - if fake.PromoteReplicaResults == nil { - return "", assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - defer func() { - if fake.PromoteReplicaPostDelays == nil { - return - } - - if delay, ok := fake.PromoteReplicaPostDelays[key]; ok { - time.Sleep(delay) - } - }() - - if fake.PromoteReplicaDelays != nil { - if delay, ok := fake.PromoteReplicaDelays[key]; ok { - select { - case <-ctx.Done(): - return "", ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.PromoteReplicaResults[key]; ok { - return result.Result, result.Error - } - - return "", assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) ReplicationStatus(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.Status, error) { - if fake.ReplicationStatusResults == nil { - return nil, assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if result, ok := fake.ReplicationStatusResults[key]; ok { - return result.Position, result.Error - } - - return nil, assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) SetMaster(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { - if fake.SetMasterResults == nil { - return assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.SetMasterDelays != nil { - if delay, ok := fake.SetMasterDelays[key]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.SetMasterResults[key]; ok { - return result - } - - return assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) SetReadWrite(ctx context.Context, tablet *topodatapb.Tablet) error { - if fake.SetReadWriteResults == nil { - return assert.AnError - } - - if tablet.Alias == nil { - return assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.SetReadWriteDelays != nil { - if delay, ok := fake.SetReadWriteDelays[key]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if err, ok := fake.SetReadWriteResults[key]; ok { - return err - } - - return assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) StopReplicationAndGetStatus(ctx context.Context, tablet *topodatapb.Tablet, mode replicationdatapb.StopReplicationMode) (*replicationdatapb.Status, *replicationdatapb.StopReplicationStatus, error) { - if fake.StopReplicationAndGetStatusResults == nil { - return nil, nil, assert.AnError - } - - if tablet.Alias == nil { - return nil, nil, assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.StopReplicationAndGetStatusDelays != nil { - if delay, ok := fake.StopReplicationAndGetStatusDelays[key]; ok { - select { - case <-ctx.Done(): - return nil, nil, ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.StopReplicationAndGetStatusResults[key]; ok { - return result.Status, result.StopStatus, result.Error - } - - return nil, nil, assert.AnError -} - -func (fake *emergencyReparenterTestTMClient) WaitForPosition(ctx context.Context, tablet *topodatapb.Tablet, position string) error { - tabletKey := topoproto.TabletAliasString(tablet.Alias) - - defer func() { - if fake.WaitForPositionPostDelays == nil { - return - } - - if delay, ok := fake.WaitForPositionPostDelays[tabletKey]; ok { - time.Sleep(delay) - } - }() - - if fake.WaitForPositionDelays != nil { - if delay, ok := fake.WaitForPositionDelays[tabletKey]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if fake.WaitForPositionResults == nil { - return assert.AnError - } - - tabletResultsByPosition, ok := fake.WaitForPositionResults[tabletKey] - if !ok { - return assert.AnError - } - - result, ok := tabletResultsByPosition[position] - if !ok { - return assert.AnError - } - - return result -} - -func (fake *emergencyReparenterTestTMClient) UndoDemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) error { - if fake.UndoDemoteMasterResults == nil { - return assert.AnError - } - - if tablet.Alias == nil { - return assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.UndoDemoteMasterDelays != nil { - if delay, ok := fake.UndoDemoteMasterDelays[key]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.UndoDemoteMasterResults[key]; ok { - return result - } - - return assert.AnError -} diff --git a/go/vt/vtctl/reparentutil/planned_reparenter_test.go b/go/vt/vtctl/reparentutil/planned_reparenter_test.go index fccf7da03cb..805e685295d 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter_test.go @@ -87,7 +87,7 @@ func TestPlannedReparenter_ReparentShard(t *testing.T) { { name: "success", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -404,9 +404,7 @@ func TestPlannedReparenter_preflightChecks(t *testing.T) { }, { name: "invariants hold with primary selection", - // (TODO:@ajm188) - Rename this type and unify it with the 4 other - // mock implementations I've written by this point. - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ ReplicationStatusResults: map[string]struct { Position *replicationdatapb.Status Error error @@ -680,7 +678,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "successful promotion", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -747,7 +745,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "cannot get snapshot of current primary", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -781,7 +779,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "primary-elect fails to catch up to current primary snapshot position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -818,7 +816,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "primary-elect times out catching up to current primary snapshot position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -860,7 +858,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "lost topology lock", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -898,7 +896,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "failed to demote current primary", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -943,7 +941,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "primary-elect fails to catch up to current primary demotion position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1000,7 +998,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "primary-elect times out catching up to current primary demotion position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1062,7 +1060,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "demotion succeeds but parent context times out", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1133,7 +1131,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "rollback fails", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1196,7 +1194,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "rollback succeeds", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1259,7 +1257,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "primary-elect fails to promote", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1324,7 +1322,7 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { { name: "promotion succeeds but parent context times out", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1469,7 +1467,7 @@ func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { }{ { name: "successful recovery", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -1494,7 +1492,7 @@ func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { }, { name: "failed to SetReadWrite", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ SetReadWriteResults: map[string]error{ "zone1-0000000100": assert.AnError, }, @@ -1509,7 +1507,7 @@ func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { }, { name: "SetReadWrite timed out", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ SetReadWriteDelays: map[string]time.Duration{ "zone1-0000000100": time.Millisecond * 50, }, @@ -1528,7 +1526,7 @@ func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { }, { name: "failed to get MasterPosition from refreshed primary", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -1552,7 +1550,7 @@ func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { }, { name: "MasterPosition timed out", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionDelays: map[string]time.Duration{ "zone1-0000000100": time.Millisecond * 50, }, @@ -1633,7 +1631,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "success", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1708,7 +1706,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "failed to DemoteMaster on a tablet", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1743,7 +1741,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "timed out during DemoteMaster on a tablet", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterDelays: map[string]time.Duration{ "zone1-0000000100": time.Millisecond * 50, }, @@ -1784,7 +1782,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "failed to DecodePosition on a tablet's demote position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1821,7 +1819,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "primary-elect not in tablet map", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{}, + tmc: &testutil.TabletManagerClient{}, unlockTopo: false, keyspace: "testkeyspace", shard: "-", @@ -1837,7 +1835,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "primary-elect not most at most advanced position", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1902,7 +1900,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "lost topology lock", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -1967,7 +1965,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "failed to promote primary-elect", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -2041,7 +2039,7 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { { name: "timed out while promoting primary-elect", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -2188,7 +2186,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "success: current primary cannot be determined", // "Case (1)" ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -2268,7 +2266,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "success: current primary is desired primary", // "Case (2)" ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -2343,7 +2341,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "success: graceful promotion", // "Case (3)" ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.MasterStatus Error error @@ -2534,7 +2532,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "promotion step fails", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ SetReadWriteResults: map[string]error{ "zone1-0000000100": assert.AnError, }, @@ -2594,7 +2592,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "lost topology lock", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -2665,7 +2663,7 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { { name: "failed to reparent tablets", ts: memorytopo.NewServer("zone1"), - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error @@ -2801,7 +2799,7 @@ func TestPlannedReparenter_reparentTablets(t *testing.T) { }{ { name: "success", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -2862,7 +2860,7 @@ func TestPlannedReparenter_reparentTablets(t *testing.T) { }, { name: "SetMaster failed on replica", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -2923,7 +2921,7 @@ func TestPlannedReparenter_reparentTablets(t *testing.T) { }, { name: "SetMaster timed out on replica", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, @@ -2990,7 +2988,7 @@ func TestPlannedReparenter_reparentTablets(t *testing.T) { }, { name: "PopulateReparentJournal failed out on new primary", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, }, @@ -3051,7 +3049,7 @@ func TestPlannedReparenter_reparentTablets(t *testing.T) { }, { name: "PopulateReparentJournal timed out on new primary", - tmc: &emergencyReparenterTestTMClient{ + tmc: &testutil.TabletManagerClient{ PopulateReparentJournalDelays: map[string]time.Duration{ "zone1-0000000100": time.Millisecond * 50, },