From 3da09af00d4243962a29134799613a8c56977f03 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 3 Apr 2020 22:12:31 +0200 Subject: [PATCH 1/2] Modify local example to use new vreplication workflows Signed-off-by: Rohit Nayak --- examples/local/101_initial_cluster.sh | 4 +- examples/local/201_customer_keyspace.sh | 20 ---------- ...mer_tablets.sh => 201_customer_tablets.sh} | 7 ++-- ...3_vertical_split.sh => 202_move_tables.sh} | 13 ++++--- ...igrate_replicas.sh => 203_switch_reads.sh} | 20 ++++++++-- ...migrate_master.sh => 204_switch_writes.sh} | 9 ++++- ...lean_commerce.sh => 205_clean_commerce.sh} | 2 +- examples/local/302_new_shards.sh | 2 - ...303_horizontal_split.sh => 303_reshard.sh} | 17 +++++---- ...igrate_replicas.sh => 304_switch_reads.sh} | 19 ++++++++-- ...migrate_master.sh => 305_switch_writes.sh} | 10 ++++- examples/local/401_teardown.sh | 2 + examples/local/env.sh | 5 ++- test/local_example.sh | 37 ++++++++++++------- 14 files changed, 98 insertions(+), 69 deletions(-) delete mode 100755 examples/local/201_customer_keyspace.sh rename examples/local/{202_customer_tablets.sh => 201_customer_tablets.sh} (74%) rename examples/local/{203_vertical_split.sh => 202_move_tables.sh} (79%) rename examples/local/{204_vertical_migrate_replicas.sh => 203_switch_reads.sh} (65%) rename examples/local/{205_vertical_migrate_master.sh => 204_switch_writes.sh} (79%) rename examples/local/{206_clean_commerce.sh => 205_clean_commerce.sh} (96%) rename examples/local/{303_horizontal_split.sh => 303_reshard.sh} (77%) rename examples/local/{304_migrate_replicas.sh => 304_switch_reads.sh} (65%) rename examples/local/{305_migrate_master.sh => 305_switch_writes.sh} (81%) diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 4472412e944..3d5b7221efa 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -21,9 +21,9 @@ source ./env.sh # start topo server if [ "${TOPO}" = "zk2" ]; then - CELL=zone1 ./scripts/zk-up.sh + CELL=zone1 ./scripts/zk-up.sh elif [ "${TOPO}" = "k8s" ]; then - CELL=zone1 ./scripts/k3s-up.sh + CELL=zone1 ./scripts/k3s-up.sh else CELL=zone1 ./scripts/etcd-up.sh fi diff --git a/examples/local/201_customer_keyspace.sh b/examples/local/201_customer_keyspace.sh deleted file mode 100755 index f948175195c..00000000000 --- a/examples/local/201_customer_keyspace.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this script creates a new keyspace in preparation for vertical resharding - -vtctlclient -server localhost:15999 CreateKeyspace -served_from='master:commerce,replica:commerce,rdonly:commerce' customer - diff --git a/examples/local/202_customer_tablets.sh b/examples/local/201_customer_tablets.sh similarity index 74% rename from examples/local/202_customer_tablets.sh rename to examples/local/201_customer_tablets.sh index ec77226342f..ec7dcbd80f0 100755 --- a/examples/local/202_customer_tablets.sh +++ b/examples/local/201_customer_tablets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ for i in 200 201 202; do done vtctlclient -server localhost:15999 InitShardMaster -force customer/0 zone1-200 -vtctlclient -server localhost:15999 CopySchemaShard -tables customer,corder commerce/0 customer/0 -vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_commerce_vsplit.json commerce -vtctlclient -server localhost:15999 ApplyVSchema -vschema_file vschema_customer_vsplit.json customer +vtctlclient -server localhost:15999 ApplyVSchema -vschema '{ "tables": { "product": {} } }' commerce +vtctlclient -server localhost:15999 ApplyVSchema -vschema '{ "tables": { "customer": {}, "corder": {} } }' customer diff --git a/examples/local/203_vertical_split.sh b/examples/local/202_move_tables.sh similarity index 79% rename from examples/local/203_vertical_split.sh rename to examples/local/202_move_tables.sh index 372c4817a90..b7ffd9699bf 100755 --- a/examples/local/203_vertical_split.sh +++ b/examples/local/202_move_tables.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,11 +19,12 @@ source ./env.sh -vtworker \ - $TOPOLOGY_FLAGS \ - -cell zone1 \ +vtctlclient \ + -server localhost:15999 \ -log_dir "$VTDATAROOT"/tmp \ -alsologtostderr \ - -use_v3_resharding_mode \ - VerticalSplitClone -min_healthy_tablets=1 -tables=customer,corder customer/0 + MoveTables \ + -workflow=commerce2customer \ + commerce customer customer,corder +sleep 2 diff --git a/examples/local/204_vertical_migrate_replicas.sh b/examples/local/203_switch_reads.sh similarity index 65% rename from examples/local/204_vertical_migrate_replicas.sh rename to examples/local/203_switch_reads.sh index 45ada0aaf63..4027267713d 100755 --- a/examples/local/204_vertical_migrate_replicas.sh +++ b/examples/local/203_switch_reads.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,20 @@ # this script migrates traffic for the new customer keyspace to the new # tablets of types rdonly and replica -vtctlclient -server localhost:15999 MigrateServedFrom customer/0 rdonly -vtctlclient -server localhost:15999 MigrateServedFrom customer/0 replica + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=rdonly \ + customer.commerce2customer + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=replica \ + customer.commerce2customer diff --git a/examples/local/205_vertical_migrate_master.sh b/examples/local/204_switch_writes.sh similarity index 79% rename from examples/local/205_vertical_migrate_master.sh rename to examples/local/204_switch_writes.sh index 211d27eb6a7..a0d5d3c31b5 100755 --- a/examples/local/205_vertical_migrate_master.sh +++ b/examples/local/204_switch_writes.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,4 +17,9 @@ # this script migrates master traffic for the customer keyspace to the # new master tablet -vtctlclient -server localhost:15999 MigrateServedFrom customer/0 master +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchWrites \ + customer.commerce2customer diff --git a/examples/local/206_clean_commerce.sh b/examples/local/205_clean_commerce.sh similarity index 96% rename from examples/local/206_clean_commerce.sh rename to examples/local/205_clean_commerce.sh index 7111a4a91fc..8bccbe99b97 100755 --- a/examples/local/206_clean_commerce.sh +++ b/examples/local/205_clean_commerce.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/local/302_new_shards.sh b/examples/local/302_new_shards.sh index c8830f97d0e..c6400bd0762 100755 --- a/examples/local/302_new_shards.sh +++ b/examples/local/302_new_shards.sh @@ -31,5 +31,3 @@ done vtctlclient -server localhost:15999 InitShardMaster -force customer/-80 zone1-300 vtctlclient -server localhost:15999 InitShardMaster -force customer/80- zone1-400 -vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/-80 -vtctlclient -server localhost:15999 CopySchemaShard customer/0 customer/80- diff --git a/examples/local/303_horizontal_split.sh b/examples/local/303_reshard.sh similarity index 77% rename from examples/local/303_horizontal_split.sh rename to examples/local/303_reshard.sh index aaa019559d4..227ea3296b2 100755 --- a/examples/local/303_horizontal_split.sh +++ b/examples/local/303_reshard.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,10 +19,11 @@ source ./env.sh -vtworker \ - $TOPOLOGY_FLAGS \ - -cell zone1 \ - -log_dir "$VTDATAROOT"/tmp \ - -alsologtostderr \ - -use_v3_resharding_mode \ - SplitClone -min_healthy_rdonly_tablets=1 customer/0 +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + Reshard \ + customer.cust2cust "0" "-80,80-" + +sleep 2 \ No newline at end of file diff --git a/examples/local/304_migrate_replicas.sh b/examples/local/304_switch_reads.sh similarity index 65% rename from examples/local/304_migrate_replicas.sh rename to examples/local/304_switch_reads.sh index 3aed031ebab..aed6e6adada 100755 --- a/examples/local/304_migrate_replicas.sh +++ b/examples/local/304_switch_reads.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,5 +16,18 @@ # this script migrates traffic for the rdonly and replica tablets -vtctlclient -server localhost:15999 MigrateServedTypes customer/0 rdonly -vtctlclient -server localhost:15999 MigrateServedTypes customer/0 replica +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=rdonly \ + customer.cust2cust + +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchReads \ + -tablet_type=replica \ + customer.cust2cust diff --git a/examples/local/305_migrate_master.sh b/examples/local/305_switch_writes.sh similarity index 81% rename from examples/local/305_migrate_master.sh rename to examples/local/305_switch_writes.sh index a9da079ea7c..34dde1f971e 100755 --- a/examples/local/305_migrate_master.sh +++ b/examples/local/305_switch_writes.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,12 @@ # this script migrates traffic for the master tablet -vtctlclient -server localhost:15999 MigrateServedTypes customer/0 master +vtctlclient \ + -server localhost:15999 \ + -log_dir "$VTDATAROOT"/tmp \ + -alsologtostderr \ + SwitchWrites \ + customer.cust2cust + # data has been copied over to shards, and databases for the new shards are now available diff --git a/examples/local/401_teardown.sh b/examples/local/401_teardown.sh index 0193f04ff9e..5c6e279fb0b 100755 --- a/examples/local/401_teardown.sh +++ b/examples/local/401_teardown.sh @@ -26,7 +26,9 @@ for tablet in 100 200 300 400; do # The zero tablet is up. Try to shutdown 0-2 tablet + mysqlctl for i in 0 1 2; do uid=$[$tablet + $i] + echo "Shutting down tablet zone1-$uid" CELL=zone1 TABLET_UID=$uid ./scripts/vttablet-down.sh + echo "Shutting down mysql zone1-$uid" CELL=zone1 TABLET_UID=$uid ./scripts/mysqlctl-down.sh done fi diff --git a/examples/local/env.sh b/examples/local/env.sh index 1d21fafc98f..74f929692f5 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -31,7 +31,7 @@ fi # mysqld might be in /usr/sbin which will not be in the default PATH PATH="/usr/sbin:$PATH" -for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld vtctl mysqlctl; do +for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions." done; @@ -54,6 +54,8 @@ if [ "${TOPO}" = "zk2" ]; then ZK_SERVER="localhost:21811,localhost:21812,localhost:21813" # shellcheck disable=SC2034 TOPOLOGY_FLAGS="-topo_implementation zk2 -topo_global_server_address ${ZK_SERVER} -topo_global_root /vitess/global" + + mkdir -p $VTDATAROOT/tmp elif [ "${TOPO}" = "k8s" ]; then # Set topology environment parameters. K8S_ADDR="localhost" @@ -68,5 +70,4 @@ else mkdir -p "${VTDATAROOT}/etcd" fi -# Create a tmp dir mkdir -p "${VTDATAROOT}/tmp" diff --git a/test/local_example.sh b/test/local_example.sh index 71c2042fbd7..ee5cd0142ec 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Vitess Authors. +# Copyright 2020 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,19 +30,30 @@ unset VTROOT # ensure that the examples can run without VTROOT now. mysql -h 127.0.0.1 -P 15306 < ../common/insert_commerce_data.sql mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql -./201_customer_keyspace.sh -./202_customer_tablets.sh -./203_vertical_split.sh -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer0_data.sql -./204_vertical_migrate_replicas.sh -./205_vertical_migrate_master.sh +./201_customer_tablets.sh + +for shard in "customer/0"; do + while true; do + mysql -h 127.0.0.1 -P 15306 "$shard" -e 'show tables' && break || echo "waiting for shard: $shard!" + sleep 1 + done; +done; + +./202_move_tables.sh + +./203_switch_reads.sh + +./204_switch_writes.sh + +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer0_data.sql # Expected to fail! mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Blacklist working as expected" -./206_clean_commerce.sh +./205_clean_commerce.sh # Expected to fail! mysql -h 127.0.0.1 -P 15306 --table < ../common/select_commerce_data.sql || echo "Tables missing as expected" + ./301_customer_sharded.sh ./302_new_shards.sh @@ -55,15 +66,13 @@ for shard in "customer/-80" "customer/80-"; do done; done; -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql -mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer80-_data.sql +./303_reshard.sh -./303_horizontal_split.sh - -./304_migrate_replicas.sh -./305_migrate_master.sh +./304_switch_reads.sh +./305_switch_writes.sh mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer-80_data.sql +mysql -h 127.0.0.1 -P 15306 --table < ../common/select_customer80-_data.sql ./401_teardown.sh From cf813a813e707c9b856b60f5d0bd17fad4eae744 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 4 Apr 2020 16:13:03 +0200 Subject: [PATCH 2/2] Deleting vschemas which are now used inline Signed-off-by: Rohit Nayak --- examples/local/vschema_commerce_vsplit.json | 5 ----- examples/local/vschema_customer_vsplit.json | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 examples/local/vschema_commerce_vsplit.json delete mode 100644 examples/local/vschema_customer_vsplit.json diff --git a/examples/local/vschema_commerce_vsplit.json b/examples/local/vschema_commerce_vsplit.json deleted file mode 100644 index 30f34172e31..00000000000 --- a/examples/local/vschema_commerce_vsplit.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "tables": { - "product": {} - } -} diff --git a/examples/local/vschema_customer_vsplit.json b/examples/local/vschema_customer_vsplit.json deleted file mode 100644 index b113a9adc9e..00000000000 --- a/examples/local/vschema_customer_vsplit.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "tables": { - "customer": {}, - "corder": {} - } -}