diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32f0a000966..b4b82473d97 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,5 @@ /docker/ @derekperkins @dkhenry /helm/ @derekperkins @dkhenry +/config/mycnf/ @morgo +/go/vt/mysqlctl/mysqld.go @morgo diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index b338743733e..0d0c7e4fab3 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -6,4 +6,4 @@ about: If you have a question, please check out our other community resources in Issues on GitHub are intended to be related to bugs or feature requests, so we recommend using our other community resources instead of asking here. - [Vitess User Guide](https://vitess.io/user-guide/introduction/) -- Any other questions can be asked in the community [Slack workspace](https://bit.ly/vitess-slack) +- Any other questions can be asked in the community [Slack workspace](https://vitess.io/slack) diff --git a/.github/bootstrap.sh b/.github/bootstrap.sh deleted file mode 100755 index 603b4310d66..00000000000 --- a/.github/bootstrap.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2164 - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a next-gen bootstrap which skips Python and Java tests, -# and does not use the VTROOT/VTTOP layout. -# -# My original intention was to use the same bootstrap.sh and gate -# for new features, but it has turned out to be difficult to do, -# due to the way that Docker cache works in the CI environment. - -function fail() { - echo "ERROR: $1" - exit 1 -} - -[[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory" - -# Create main directories. - -VTROOT="$PWD" - -mkdir -p dist -mkdir -p bin -mkdir -p lib -mkdir -p vthook - -source ./dev.env - -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -goversion_min 1.12 || fail "Go is not version 1.12+" - -# Set up required soft links. -# TODO(mberlin): Which of these can be deleted? -ln -snf "$VTROOT/py" "$VTROOT/py-vtdb" -ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" -ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh" -ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" -ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" - -# git hooks are only required if someone intends to contribute. - -echo "creating git hooks" -mkdir -p "$VTROOT/.git/hooks" -ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit" -ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg" -git config core.hooksPath "$VTROOT/.git/hooks" - -# install_dep is a helper function to generalize the download and installation of dependencies. -# -# If the installation is successful, it puts the installed version string into -# the $dist/.installed_version file. If the version has not changed, bootstrap -# will skip future installations. -function install_dep() { - if [[ $# != 4 ]]; then - fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*" - fi - local name="$1" - local version="$2" - local dist="$3" - local install_func="$4" - - version_file="$dist/.installed_version" - if [[ -f "$version_file" && "$(cat "$version_file")" == "$version" ]]; then - echo "skipping $name install. remove $dist to force re-install." - return - fi - - echo "installing $name $version" - - # shellcheck disable=SC2064 - trap "fail '$name build failed'; exit 1" ERR - - # Cleanup any existing data and re-create the directory. - rm -rf "$dist" - mkdir -p "$dist" - - # Change $CWD to $dist before calling "install_func". - pushd "$dist" >/dev/null - # -E (same as "set -o errtrace") makes sure that "install_func" inherits the - # trap. If here's an error, the trap will be called which will exit this - # script. - set -E - $install_func "$version" "$dist" - set +E - popd >/dev/null - - trap - ERR - - echo "$version" > "$version_file" -} - - -# -# 1. Installation of dependencies. -# - -# Wrapper around the `arch` command which plays nice with OS X -function get_arch() { - case $(uname) in - Linux) arch;; - Darwin) uname -m;; - esac -} - -# Install protoc. -function install_protoc() { - local version="$1" - local dist="$2" - - case $(uname) in - Linux) local platform=linux;; - Darwin) local platform=osx;; - esac - - case $(get_arch) in - aarch64) local target=aarch_64;; - x86_64) local target=x86_64;; - *) echo "ERROR: unsupported architecture"; exit 1;; - esac - - wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip - unzip "protoc-$version-$platform-${target}.zip" - ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc" -} -protoc_ver=3.6.1 -install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc - -# Download and install etcd, link etcd binary into our root. -function install_etcd() { - local version="$1" - local dist="$2" - - case $(uname) in - Linux) local platform=linux; local ext=tar.gz;; - Darwin) local platform=darwin; local ext=zip;; - esac - - case $(get_arch) in - aarch64) local target=arm64;; - x86_64) local target=amd64;; - *) echo "ERROR: unsupported architecture"; exit 1;; - esac - - download_url=https://github.com/coreos/etcd/releases/download - file="etcd-${version}-${platform}-${target}.${ext}" - - wget "$download_url/$version/$file" - if [ "$ext" = "tar.gz" ]; then - tar xzf "$file" - else - unzip "$file" - fi - rm "$file" - ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd" -} -install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd - -echo -echo "bootstrap finished" diff --git a/.github/workflows/check_make_parser.yml b/.github/workflows/check_make_parser.yml new file mode 100644 index 00000000000..0657236fc38 --- /dev/null +++ b/.github/workflows/check_make_parser.yml @@ -0,0 +1,34 @@ +name: check_make_parser +on: [push, pull_request] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: check_make_parser + run: | + tools/check_make_parser.sh + diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml new file mode 100644 index 00000000000..a2f3f9d7d3d --- /dev/null +++ b/.github/workflows/cluster_endtoend.yml @@ -0,0 +1,33 @@ +name: cluster_endtoend +on: [push, pull_request] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: cluster_endtoend + run: | + make e2e_test_cluster diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml new file mode 100644 index 00000000000..0439d067116 --- /dev/null +++ b/.github/workflows/e2e_race.yml @@ -0,0 +1,33 @@ +name: e2e_race +on: [push, pull_request] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: e2e_race + run: | + make e2e_test_race diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml new file mode 100644 index 00000000000..6ebec31c3b5 --- /dev/null +++ b/.github/workflows/endtoend.yml @@ -0,0 +1,37 @@ +name: endtoend +on: [push, pull_request] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + run: | + make minimaltools + + - name: Build + run: | + make build + + - name: endtoend + run: | + tools/e2e_test_runner.sh diff --git a/.github/workflows/local-example.yml b/.github/workflows/local_example.yml similarity index 65% rename from .github/workflows/local-example.yml rename to .github/workflows/local_example.yml index 03a358d8161..fe0035ec10e 100644 --- a/.github/workflows/local-example.yml +++ b/.github/workflows/local_example.yml @@ -1,4 +1,4 @@ -name: Local Example +name: local_example on: [push, pull_request] jobs: @@ -24,18 +24,15 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh + make minimaltools - name: Build run: | - GOBIN=$PWD/bin make build + make build - - name: Run Local Example + - name: local_example run: | - export PATH=$PWD/bin:$PATH - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD test/local_example.sh + test/local_example.sh diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml new file mode 100644 index 00000000000..dd53b59a1d6 --- /dev/null +++ b/.github/workflows/unit.yml @@ -0,0 +1,33 @@ +name: unit +on: [push, pull_request] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.12 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget ant openjdk-8-jdk + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make tools + run: | + make tools + + - name: unit + run: | + make test diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml new file mode 100644 index 00000000000..a85a6c87ef5 --- /dev/null +++ b/.github/workflows/unit_race.yml @@ -0,0 +1,33 @@ +name: unit_race +on: [push] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.12 + + - name: Check out code + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make tools + run: | + make tools + + - name: unit_race + run: | + make unit_test_race diff --git a/.gitignore b/.gitignore index 86b6f1b3850..2561bdd9379 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,10 @@ releases # Vagrant .vagrant + +dist/* +py-vtdb* +vthook* +bin* + +vtdataroot* diff --git a/MAINTAINERS.md b/MAINTAINERS.md index a11a8e6fc83..6eb672ab6a2 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -10,10 +10,10 @@ The following is the full list, alphabetically ordered. * Derek Perkins ([derekperkins](https://github.com/derekperkins)) derek@nozzle.io * Harshit Gangal ([harshit-gangal](https://github.com/harshit-gangal)) harshit.gangal@gmail.com * Jon Tirsen ([tirsen](https://github.com/tirsen)) jontirsen@squareup.com -* Leo X. Lin ([leoxlin](https://github.com/leoxlin)) llin@hubspot.com * Michael Demmer ([demmer](https://github.com/demmer)) mdemmer@slack-corp.com * Michael Pawliszyn ([mpawliszyn](https://github.com/mpawliszyn)) mikepaw@squareup.com * Morgan Tocker ([morgo](https://github.com/morgo)) morgan@planetscale.com +* Paul Hemberger ([pH14](https://github.com/pH14)) phemberger@hubspot.com * Rafael Chacon ([rafael](https://github.com/rafael)) rchacon@slack-corp.com * Sugu Sougoumarane ([sougou](https://github.com/sougou)) sougou@planetscale.com @@ -35,7 +35,7 @@ sougou, dweitzman, deepthi, systay deepthi, rafael, enisoc ### Java -mpawliszyn, leoxlin, harshit-gangal +mpawliszyn, pH14, harshit-gangal ### Kubernetes derekperkins, dkhenry, enisoc diff --git a/Makefile b/Makefile index d0581272a65..04b3c121e3f 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,15 @@ MAKEFLAGS = -s +export GOBIN=$(PWD)/bin +export GO111MODULE=on +export GODEBUG=tls13=0 + # Disabled parallel processing of target prerequisites to avoid that integration tests are racing each other (e.g. for ports) and may fail. # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time. .NOTPARALLEL: -.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race +.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools all: build @@ -33,14 +37,6 @@ ifdef VT_EXTRA_BUILD_FLAGS export EXTRA_BUILD_FLAGS := $(VT_EXTRA_BUILD_FLAGS) endif -# Link against the MySQL library in $VT_MYSQL_ROOT if it's specified. -ifdef VT_MYSQL_ROOT -# Clutter the env var only if it's a non-standard path. - ifneq ($(VT_MYSQL_ROOT),/usr) - CGO_LDFLAGS += -L$(VT_MYSQL_ROOT)/lib - endif -endif - build_web: echo $$(date): Building web artifacts cd web/vtctld2 && ng build -prod @@ -50,6 +46,7 @@ build: ifndef NOBANNER echo $$(date): Building source tree endif + bash ./build.env go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... parser: @@ -58,8 +55,9 @@ parser: # To pass extra flags, run test.go manually. # For example: go run test.go -docker=false -- --extra-flag # For more info see: go run test.go -help -test: - go run test.go -docker=false +test: build dependency_check + echo $$(date): Running unit tests + tools/unit_test_runner.sh site_test: unit_test site_integration_test @@ -68,25 +66,13 @@ clean: rm -rf third_party/acolyte rm -rf go/vt/.proto.tmp -# This will remove object files for all Go projects in the same GOPATH. -# This is necessary, for example, to make sure dependencies are rebuilt -# when switching between different versions of Go. -clean_pkg: - rm -rf ../../../../pkg Godeps/_workspace/pkg - # Remove everything including stuff pulled down by bootstrap.sh cleanall: - # symlinks - for f in config data py-vtdb; do test -L ../../../../$$f && rm ../../../../$$f; done # directories created by bootstrap.sh # - exclude vtdataroot and vthook as they may have data we want - rm -rf ../../../../bin ../../../../dist ../../../../lib ../../../../pkg + rm -rf bin dist lib pkg # Remind people to run bootstrap.sh again - echo "Please run bootstrap.sh again to setup your environment" - -unit_test: build - echo $$(date): Running unit tests - go test $(VT_GO_PARALLEL) ./go/... + echo "Please run 'make tools' again to setup your environment" e2e_test: build echo $$(date): Running endtoend tests @@ -98,12 +84,15 @@ e2e_test: build unit_test_cover: build go test $(VT_GO_PARALLEL) -cover ./go/... | misc/parse_cover.py -unit_test_race: build +unit_test_race: build dependency_check tools/unit_test_race.sh e2e_test_race: build tools/e2e_test_race.sh +e2e_test_cluster: build + tools/e2e_test_cluster.sh + .ONESHELL: SHELL = /bin/bash @@ -117,7 +106,7 @@ site_integration_test: java_test: go install ./go/cmd/vtgateclienttest ./go/cmd/vtcombo - mvn -f java/pom.xml -B clean verify + VTROOT=${PWD} mvn -f java/pom.xml -B clean verify install_protoc-gen-go: go install github.com/golang/protobuf/protoc-gen-go @@ -246,9 +235,6 @@ docker_lite_alpine: docker_guestbook: cd examples/kubernetes/guestbook && ./build.sh -docker_publish_site: - docker build -f docker/publish-site/Dockerfile -t vitess/publish-site . - # This rule loads the working copy of the code into a bootstrap image, # and then runs the tests inside Docker. # Example: $ make docker_test flavor=mariadb @@ -286,3 +272,14 @@ packages: docker_base docker build -f docker/packaging/Dockerfile -t vitess/packaging . docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm + +tools: + echo $$(date): Installing dependencies + BUILD_PYTHON=0 ./bootstrap.sh + +minimaltools: + echo $$(date): Installing minimal dependencies + BUILD_PYTHON=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh + +dependency_check: + ./tools/dependency_check.sh diff --git a/README.md b/README.md index f8b14b0c126..bbe10c4c8c7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Ask questions in the discussion forum. For topics that are better discussed live, please join the -[Vitess Slack](https://bit.ly/vitess-slack) workspace. +[Vitess Slack](https://vitess.io/slack) workspace. Subscribe to [vitess-announce@googlegroups.com](https://groups.google.com/forum/#!forum/vitess-announce) diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 03a4ca827af..00000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,21 +0,0 @@ -pr: -- master - -pool: - vmImage: 'ubuntu-latest' - -variables: - flags: "-docker -use_docker_cache -timeout=8m -print-log" - shards: [0, 1, 2, 3, 4] - flavors: ["mysql56", "mysql57", "mysql80", "mariadb", "mariadb103", "percona57", "percona80"] -jobs: -- job: tests - strategy: - matrix: - ${{ each flavor in variables.flavors }}: - ${{ each shard in variables.shards }}: - ${{ format('{0}{1}', flavor, shard) }}: - flavor: ${{ flavor }} - shard: ${{ shard }} - steps: - - script: go run test.go -shard $(shard) -flavor $(flavor) $(flags) diff --git a/bootstrap.sh b/bootstrap.sh index 20fef115453..c943c0d8bbd 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -15,69 +15,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +### This file is executed by 'make tools'. You do not need to execute it directly. + +source ./dev.env # Outline of this file. # 0. Initialization and helper methods. # 1. Installation of dependencies. -BUILD_TESTS=${BUILD_TESTS:-1} BUILD_PYTHON=${BUILD_PYTHON:-1} BUILD_JAVA=${BUILD_JAVA:-1} +BUILD_CONSUL=${BUILD_CONSUL:-1} # # 0. Initialization and helper methods. # -function fail() { - echo "ERROR: $1" - exit 1 -} - [[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory" -# Create main directories. -VTROOT="${VTROOT:-${PWD/\/src\/vitess.io\/vitess/}}" -mkdir -p "$VTROOT/dist" -mkdir -p "$VTROOT/bin" -mkdir -p "$VTROOT/lib" -mkdir -p "$VTROOT/vthook" - -# This is required for VIRTUALENV -# Used by Python below - -if [ "$BUILD_TESTS" == 1 ] ; then - source ./dev.env -else - source ./build.env -fi - -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -goversion_min 1.12 || fail "Go is not version 1.12+" - -if [ "$BUILD_TESTS" == 1 ] ; then - # Set up required soft links. - # TODO(mberlin): Which of these can be deleted? - ln -snf "$VTTOP/config" "$VTROOT/config" - ln -snf "$VTTOP/data" "$VTROOT/data" - ln -snf "$VTTOP/py" "$VTROOT/py-vtdb" - ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" - ln -snf "$VTTOP/test/vthook-test.sh" "$VTROOT/vthook/test.sh" - ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" - ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" -else - ln -snf "$VTTOP/config" "$VTROOT/config" - ln -snf "$VTTOP/data" "$VTROOT/data" - ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" -fi - -# git hooks are only required if someone intends to contribute. - -echo "creating git hooks" -mkdir -p "$VTTOP/.git/hooks" -ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit" -ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg" -(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks") - # install_dep is a helper function to generalize the download and installation of dependencies. # # If the installation is successful, it puts the installed version string into @@ -127,12 +82,11 @@ function install_dep() { # 1. Installation of dependencies. # -# Wrapper around the `arch` command which plays nice with OS X +# We should not use the arch command, since it is not reliably +# available on macOS or some linuxes: +# https://www.gnu.org/software/coreutils/manual/html_node/arch-invocation.html function get_arch() { - case $(uname) in - Linux) arch;; - Darwin) uname -m;; - esac + uname -m } @@ -235,8 +189,9 @@ function install_etcd() { fi rm "$file" ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd" + ln -snf "$dist/etcd-${version}-${platform}-${target}/etcdctl" "$VTROOT/bin/etcdctl" } -install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd +command -v etcd && echo "etcd already installed" || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd # Download and install consul, link consul binary into our root. @@ -260,8 +215,10 @@ function install_consul() { unzip "consul_${version}_${platform}_${target}.zip" ln -snf "$dist/consul" "$VTROOT/bin/consul" } -install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul +if [ "$BUILD_CONSUL" == 1 ] ; then + install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul +fi # Install py-mock. function install_pymock() { @@ -273,7 +230,7 @@ function install_pymock() { PYTHONPATH=$(prepend_path "$PYTHONPATH" "$dist/lib/python2.7/site-packages") export PYTHONPATH - pushd "$VTTOP/third_party/py" >/dev/null + pushd "$VTROOT/third_party/py" >/dev/null tar -xzf "mock-$version.tar.gz" cd "mock-$version" $PYTHON ./setup.py install --prefix="$dist" @@ -306,9 +263,25 @@ function install_chromedriver() { local version="$1" local dist="$2" - curl -sL "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip" > chromedriver_linux64.zip - unzip -o -q chromedriver_linux64.zip -d "$dist" - rm chromedriver_linux64.zip + if [ "$(arch)" == "aarch64" ] ; then + os=$(cat /etc/*release | grep "^ID=" | cut -d '=' -f 2) + case $os in + ubuntu|debian) + sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6 + ;; + centos|fedora) + sudo yum update -y && yum install -y libX11 unzip wget + ;; + esac + echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126" + wget https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip + unzip -o -q chromedriver-v6.0.3-linux-arm64.zip -d "$dist" + rm chromedriver-v6.0.3-linux-arm64.zip + else + curl -sL "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip" > chromedriver_linux64.zip + unzip -o -q chromedriver_linux64.zip -d "$dist" + rm chromedriver_linux64.zip + fi } if [ "$BUILD_PYTHON" == 1 ] ; then install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver @@ -319,4 +292,4 @@ if [ "$BUILD_PYTHON" == 1 ] ; then fi echo -echo "bootstrap finished - run 'source dev.env' or 'source build.env' in your shell before building." +echo "bootstrap finished - run 'make build' to compile" diff --git a/build.env b/build.env old mode 100644 new mode 100755 index 6bc36b42206..467749a10f1 --- a/build.env +++ b/build.env @@ -14,27 +14,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Plese ensure dev.env is written in a way which is POSIX (bourne) -# shell compatible. -# - Some build systems like rpm require the different scriptlets used -# to build a package to be run under a POSIX shell so non-POSIX -# syntax will break that as dev.env will not be sourced by bash.. - -# Import prepend_path function. -dir="$(dirname "${BASH_SOURCE[0]}")" -# shellcheck source=tools/shell_functions.inc -if ! source "${dir}/tools/shell_functions.inc"; then - echo "failed to load tools/shell_functions.inc" - return 1 -fi - -VTTOP=$(pwd) -export VTTOP -VTROOT="${VTROOT:-${VTTOP/\/src\/vitess.io\/vitess/}}" -export VTROOT -# VTTOP sanity check -if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then - echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess" -fi - -export GO111MODULE=on +source ./tools/shell_functions.inc + +go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions." +goversion_min 1.12 || fail "Go is not version 1.12+. See https://vitess.io/contributing/build-from-source for install instructions." + +mkdir -p dist +mkdir -p bin +mkdir -p lib +mkdir -p vthook + +export VTROOT="$PWD" +export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" +export PATH="$PWD/bin:$PATH" + +mkdir -p "$VTDATAROOT" + +# Set up required soft links. +# TODO(mberlin): Which of these can be deleted? +ln -snf "$PWD/py" py-vtdb +ln -snf "$PWD/go/vt/zkctl/zksrv.sh" bin/zksrv.sh +ln -snf "$PWD/test/vthook-test.sh" vthook/test.sh +ln -snf "$PWD/test/vthook-test_backup_error" vthook/test_backup_error +ln -snf "$PWD/test/vthook-test_backup_transform" vthook/test_backup_transform + +# install git hooks + +mkdir -p .git/hooks +ln -sf "$PWD/misc/git/pre-commit" .git/hooks/pre-commit +ln -sf "$PWD/misc/git/commit-msg" .git/hooks/commit-msg +git config core.hooksPath .git/hooks diff --git a/config/init_db.sql b/config/init_db.sql index 5b56939c3c3..836c8c997e6 100644 --- a/config/init_db.sql +++ b/config/init_db.sql @@ -84,6 +84,13 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'localhost'; +# User for general MySQL monitoring. +CREATE USER 'vt_monitoring'@'localhost'; +GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD + ON *.* TO 'vt_monitoring'@'localhost'; +GRANT SELECT, UPDATE, DELETE, DROP + ON performance_schema.* TO 'vt_monitoring'@'localhost'; + # User for Orchestrator (https://github.com/github/orchestrator). CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password'; GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD diff --git a/config/mycnf/default-fast.cnf b/config/mycnf/default-fast.cnf index 969b51baa34..e3d852fcfc0 100644 --- a/config/mycnf/default-fast.cnf +++ b/config/mycnf/default-fast.cnf @@ -1,37 +1,24 @@ -# basic config parameters for all db instances in the grid +# This sets some unsafe settings specifically for +# the test-suite which is currently MySQL 5.7 based +# In future it should be renamed testsuite.cnf -sql_mode = STRICT_TRANS_TABLES -character_set_server = utf8 -collation_server = utf8_general_ci -connect_timeout = 30 -datadir = {{.DataDir}} -expire_logs_days = 3 -innodb_buffer_pool_size = 64M -innodb_data_home_dir = {{.InnodbDataHomeDir}} -innodb_flush_log_at_trx_commit = 2 -innodb_lock_wait_timeout = 20 +innodb_buffer_pool_size = 32M +innodb_flush_log_at_trx_commit = 0 innodb_log_buffer_size = 1M -innodb_log_file_size = 4M -innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}} +innodb_log_file_size = 5M + +# Native AIO tends to run into aio-max-nr limit during test startup. +innodb_use_native_aio = 0 + key_buffer_size = 2M -log-error = {{.ErrorLogPath}} -long_query_time = 2 -pid-file = {{.PidFile}} -port = {{.MysqlPort}} -# all db instances should start in read-only mode - once the db is started and -# fully functional, we'll push it into read-write mode -read-only -server-id = {{.ServerID}} -skip-name-resolve -# we now need networking for replication. this is a tombstone to simpler times. -#skip_networking -# all db instances should skip the slave startup - that way we can do any -# out-of-bounds checking before we restart everything - in case we need to do -# some extra work to skip mangled transactions or fudge the slave start -skip_slave_start -slave_net_timeout = 60 -slave_load_tmpdir = {{.SlaveLoadTmpDir}} -slow-query-log -slow-query-log-file = {{.SlowLogPath}} -socket = {{.SocketFile}} -tmpdir = {{.TmpDir}} +sync_binlog=0 +innodb_doublewrite=0 + +# These two settings are required for the testsuite to pass, +# but enabling them does not spark joy. They should be removed +# in the future. See: +# https://github.com/vitessio/vitess/issues/5395 +# https://github.com/vitessio/vitess/issues/5396 + +binlog-format=statement +sql_mode = STRICT_TRANS_TABLES diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf index 61f767a8032..b8af15801b0 100644 --- a/config/mycnf/default.cnf +++ b/config/mycnf/default.cnf @@ -1,35 +1,35 @@ -# basic config parameters for all db instances in the grid +# Global configuration that is auto-included for all MySQL/MariaDB versions -sql_mode = STRICT_TRANS_TABLES -binlog_format = statement -character_set_server = utf8 -collation_server = utf8_general_ci -connect_timeout = 30 datadir = {{.DataDir}} -expire_logs_days = 3 -innodb_buffer_pool_size = 32M innodb_data_home_dir = {{.InnodbDataHomeDir}} -innodb_flush_log_at_trx_commit = 2 -innodb_lock_wait_timeout = 20 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}} log-error = {{.ErrorLogPath}} -long_query_time = 2 -max_allowed_packet = 64M -max_connections = 500 pid-file = {{.PidFile}} port = {{.MysqlPort}} + # all db instances should start in read-only mode - once the db is started and # fully functional, we'll push it into read-write mode read-only server-id = {{.ServerID}} -skip-name-resolve + # all db instances should skip the slave startup - that way we can do any # additional configuration (like enabling semi-sync) before we connect to # the master. skip_slave_start -slave_net_timeout = 60 slave_load_tmpdir = {{.SlaveLoadTmpDir}} -slow-query-log -slow-query-log-file = {{.SlowLogPath}} socket = {{.SocketFile}} tmpdir = {{.TmpDir}} + +slow-query-log-file = {{.SlowLogPath}} + +# These are sensible defaults that apply to all MySQL/MariaDB versions + +long_query_time = 2 +slow-query-log +skip-name-resolve +connect_timeout = 30 +innodb_lock_wait_timeout = 20 +max_allowed_packet = 64M +max_connections = 500 + + diff --git a/config/mycnf/master.cnf b/config/mycnf/master.cnf deleted file mode 100644 index 481f06f5ffd..00000000000 --- a/config/mycnf/master.cnf +++ /dev/null @@ -1,5 +0,0 @@ -# master.cnf parameters - -log-bin = {{.BinLogPath}} -log-slave-updates -sync_binlog = 1 diff --git a/config/mycnf/master_mariadb100.cnf b/config/mycnf/master_mariadb100.cnf index 3eed4d8ea5f..544681d601d 100644 --- a/config/mycnf/master_mariadb100.cnf +++ b/config/mycnf/master_mariadb100.cnf @@ -1,7 +1,5 @@ # This file is auto-included when MariaDB 10.0 is detected. -innodb_support_xa = 0 - # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's # available if desired, but it's disabled at startup. @@ -11,6 +9,11 @@ innodb_support_xa = 0 # promoted or demoted. plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so +slave_net_timeout = 60 + +# MariaDB 10.0 is unstrict by default +sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION + # enable strict mode so it's safe to compare sequence numbers across different server IDs. gtid_strict_mode = 1 innodb_stats_persistent = 0 @@ -21,3 +24,21 @@ innodb_stats_persistent = 0 # a master that becomes unresponsive. rpl_semi_sync_master_timeout = 1000000000000000000 rpl_semi_sync_master_wait_no_slave = 1 + + +character_set_server = utf8 +collation_server = utf8_general_ci + +expire_logs_days = 3 + +log_bin +sync_binlog = 1 +binlog_format = ROW +log_slave_updates +expire_logs_days = 3 + +# In MariaDB the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci + diff --git a/config/mycnf/master_mariadb101.cnf b/config/mycnf/master_mariadb101.cnf index 8c5b9e47fac..01567b4d960 100644 --- a/config/mycnf/master_mariadb101.cnf +++ b/config/mycnf/master_mariadb101.cnf @@ -1,7 +1,5 @@ # This file is auto-included when MariaDB 10.1 is detected. -innodb_support_xa = 0 - # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's # available if desired, but it's disabled at startup. @@ -11,6 +9,11 @@ innodb_support_xa = 0 # promoted or demoted. plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so +slave_net_timeout = 60 + +# MariaDB 10.1 default is only no-engine-substitution and no-auto-create-user +sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_CREATE_USER + # enable strict mode so it's safe to compare sequence numbers across different server IDs. gtid_strict_mode = 1 innodb_stats_persistent = 0 @@ -21,3 +24,20 @@ innodb_stats_persistent = 0 # a master that becomes unresponsive. rpl_semi_sync_master_timeout = 1000000000000000000 rpl_semi_sync_master_wait_no_slave = 1 + + +character_set_server = utf8 +collation_server = utf8_general_ci + +expire_logs_days = 3 + +log_bin +sync_binlog = 1 +binlog_format = ROW +log_slave_updates +expire_logs_days = 3 + +# In MariaDB the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci diff --git a/config/mycnf/master_mariadb102.cnf b/config/mycnf/master_mariadb102.cnf index 1ea39a2a47c..17d83cc3610 100644 --- a/config/mycnf/master_mariadb102.cnf +++ b/config/mycnf/master_mariadb102.cnf @@ -1,7 +1,5 @@ # This file is auto-included when MariaDB 10.2 is detected. -innodb_support_xa = 0 - # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's # available if desired, but it's disabled at startup. @@ -21,3 +19,20 @@ innodb_stats_persistent = 0 # a master that becomes unresponsive. rpl_semi_sync_master_timeout = 1000000000000000000 rpl_semi_sync_master_wait_no_slave = 1 + + +character_set_server = utf8 +collation_server = utf8_general_ci + +expire_logs_days = 3 + +log_bin +sync_binlog = 1 +binlog_format = ROW +log_slave_updates +expire_logs_days = 3 + +# In MariaDB the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci diff --git a/config/mycnf/master_mariadb103.cnf b/config/mycnf/master_mariadb103.cnf index ac8b38404fd..36eef4f2f50 100644 --- a/config/mycnf/master_mariadb103.cnf +++ b/config/mycnf/master_mariadb103.cnf @@ -4,20 +4,28 @@ gtid_strict_mode = 1 innodb_stats_persistent = 0 -# Semi-sync replication is required for automated unplanned failover -# (when the master goes away). Here we just load the plugin so it's -# available if desired, but it's disabled at startup. -# -# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync -# at the proper time when replication is set up, or when masters are -# promoted or demoted. - -# semi_sync has been merged into master as of mariadb 10.3 so this is no longer needed -#plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so - # When semi-sync is enabled, don't allow fallback to async # if you get no ack, or have no slaves. This is necessary to # prevent alternate futures when doing a failover in response to # a master that becomes unresponsive. rpl_semi_sync_master_timeout = 1000000000000000000 rpl_semi_sync_master_wait_no_slave = 1 + + +character_set_server = utf8 +collation_server = utf8_general_ci + +expire_logs_days = 3 + +log_bin +sync_binlog = 1 +binlog_format = ROW +log_slave_updates +expire_logs_days = 3 + +# In MariaDB the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci + + diff --git a/config/mycnf/master_mariadb104.cnf b/config/mycnf/master_mariadb104.cnf new file mode 100644 index 00000000000..9444e8d67fa --- /dev/null +++ b/config/mycnf/master_mariadb104.cnf @@ -0,0 +1,31 @@ +# This file is auto-included when MariaDB 10.4 is detected. + +# enable strict mode so it's safe to compare sequence numbers across different server IDs. +gtid_strict_mode = 1 +innodb_stats_persistent = 0 + +# When semi-sync is enabled, don't allow fallback to async +# if you get no ack, or have no slaves. This is necessary to +# prevent alternate futures when doing a failover in response to +# a master that becomes unresponsive. +rpl_semi_sync_master_timeout = 1000000000000000000 +rpl_semi_sync_master_wait_no_slave = 1 + + +character_set_server = utf8 +collation_server = utf8_general_ci + +expire_logs_days = 3 + +log_bin +sync_binlog = 1 +binlog_format = ROW +log_slave_updates +expire_logs_days = 3 + +# In MariaDB the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci + + diff --git a/config/mycnf/master_mysql56.cnf b/config/mycnf/master_mysql56.cnf index dcb8a4e113f..7524ef1663a 100644 --- a/config/mycnf/master_mysql56.cnf +++ b/config/mycnf/master_mysql56.cnf @@ -1,20 +1,29 @@ # This file is auto-included when MySQL 5.6 is detected. -# Options for enabling GTID -# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html -gtid_mode = ON +# MySQL 5.6 does not enable the binary log by default, and +# the default for sync_binlog is unsafe. The format is TABLE, and +# info repositories also default to file. + log_bin +sync_binlog = 1 +gtid_mode = ON +binlog_format = ROW log_slave_updates enforce_gtid_consistency - -# Crash-safe replication settings. +expire_logs_days = 3 master_info_repository = TABLE relay_log_info_repository = TABLE relay_log_purge = 1 relay_log_recovery = 1 +slave_net_timeout = 60 -# Native AIO tends to run into aio-max-nr limit during test startup. -innodb_use_native_aio = 0 +# In MySQL 5.6 the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci + +# MySQL 5.6 is unstrict by default +sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's @@ -31,3 +40,4 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy # a master that becomes unresponsive. rpl_semi_sync_master_timeout = 1000000000000000000 rpl_semi_sync_master_wait_no_slave = 1 + diff --git a/config/mycnf/master_mysql57.cnf b/config/mycnf/master_mysql57.cnf index 381b05ac14c..82c4e36c5fb 100644 --- a/config/mycnf/master_mysql57.cnf +++ b/config/mycnf/master_mysql57.cnf @@ -1,19 +1,23 @@ # This file is auto-included when MySQL 5.7 is detected. -# Options for enabling GTID -# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html +# MySQL 5.7 does not enable the binary log by default, and +# info repositories default to file + gtid_mode = ON log_bin log_slave_updates enforce_gtid_consistency -innodb_use_native_aio = 0 - -# Crash-safe replication settings. +expire_logs_days = 3 master_info_repository = TABLE relay_log_info_repository = TABLE relay_log_purge = 1 relay_log_recovery = 1 +# In MySQL 5.7 the default charset is latin1 + +character_set_server = utf8 +collation_server = utf8_general_ci + # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's # available if desired, but it's disabled at startup. diff --git a/config/mycnf/master_mysql80.cnf b/config/mycnf/master_mysql80.cnf index e92b794ef9b..6c3d77d5135 100644 --- a/config/mycnf/master_mysql80.cnf +++ b/config/mycnf/master_mysql80.cnf @@ -1,20 +1,18 @@ # This file is auto-included when MySQL 8.0 is detected. -# Options for enabling GTID -# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html +# MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories +# It does not enable GTIDs or enforced GTID consistency + gtid_mode = ON -log_bin -log_slave_updates enforce_gtid_consistency - -# Crash-safe replication settings. -master_info_repository = TABLE -relay_log_info_repository = TABLE -relay_log_purge = 1 relay_log_recovery = 1 +binlog_expire_logs_seconds = 259200 + +# disable mysqlx +mysqlx = 0 -# Native AIO tends to run into aio-max-nr limit during test startup. -innodb_use_native_aio = 0 +# 8.0 changes the default auth-plugin to caching_sha2_password +default_authentication_plugin = mysql_native_password # Semi-sync replication is required for automated unplanned failover # (when the master goes away). Here we just load the plugin so it's @@ -25,16 +23,9 @@ innodb_use_native_aio = 0 # promoted or demoted. plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so -# When semi-sync is enabled, don't allow fallback to async -# if you get no ack, or have no slaves. This is necessary to -# prevent alternate futures when doing a failover in response to -# a master that becomes unresponsive. -rpl_semi_sync_master_timeout = 1000000000000000000 -rpl_semi_sync_master_wait_no_slave = 1 - -# disable mysqlx -mysqlx = 0 +# MySQL 8.0 will not load plugins during --initialize +# which makes these options unknown. Prefixing with --loose +# tells the server it's fine if they are not understood. +loose_rpl_semi_sync_master_timeout = 1000000000000000000 +loose_rpl_semi_sync_master_wait_no_slave = 1 -# 8.0 changes the default auth-plugin to caching_sha2_password -default_authentication_plugin = mysql_native_password -secure_file_priv = NULL diff --git a/config/mycnf/rbr.cnf b/config/mycnf/rbr.cnf index 5dde64cda57..10c17a6f09c 100644 --- a/config/mycnf/rbr.cnf +++ b/config/mycnf/rbr.cnf @@ -1 +1,3 @@ -binlog-format=row +# This file is used to allow legacy tests to pass +# In theory it should not be required +binlog_format=row diff --git a/config/mycnf/replica.cnf b/config/mycnf/replica.cnf deleted file mode 100644 index 74e9f2b34ea..00000000000 --- a/config/mycnf/replica.cnf +++ /dev/null @@ -1,13 +0,0 @@ -# replica.cnf - reserved for future tuning - -relay-log = {{.RelayLogPath}} -relay-log-index = {{.RelayLogIndexPath}} -relay-log-info-file = {{.RelayLogInfoPath}} -master-info-file = {{.MasterInfoFile}} - -# required if this master is chained -# probably safe to turn on all the time at the expense of some disk I/O -# note: this is in the master conf too -log-slave-updates - -#slave_compressed_protocol diff --git a/config/mycnf/vtcombo.cnf b/config/mycnf/vtcombo.cnf deleted file mode 100644 index de6141f2c97..00000000000 --- a/config/mycnf/vtcombo.cnf +++ /dev/null @@ -1 +0,0 @@ -max_connections = 5000 diff --git a/dev.env b/dev.env index 1c0d1ebdd59..f256abbd93a 100644 --- a/dev.env +++ b/dev.env @@ -22,12 +22,6 @@ source ./build.env -export GOTOP=$VTTOP/go -export PYTOP=$VTTOP/py - -export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" -mkdir -p "$VTDATAROOT" - export VTPORTSTART=15000 # Add all site-packages or dist-packages directories below $VTROOT/dist to $PYTHONPATH. @@ -43,8 +37,8 @@ IFS="$BACKUP_IFS" PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/py-vtdb") PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/dist/selenium") -PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test") -PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test/cluster/sandbox") +PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test") +PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test/cluster/sandbox") export PYTHONPATH # Ensure bootstrap.sh uses python2 on systems which default to python3. @@ -62,22 +56,8 @@ PATH=$(prepend_path "$PATH" "$VTROOT/dist/chromedriver") PATH=$(prepend_path "$PATH" "$VTROOT/dist/node/bin") export PATH -# mysql install location. Please set based on your environment. -# Build will not work if this is incorrect. - -if [[ "$VT_MYSQL_ROOT" == "" ]]; then - if [[ "$(which mysql)" == "" ]]; then - echo "WARNING: VT_MYSQL_ROOT unset because mysql not found. Did you install a client package?" - else - VT_MYSQL_ROOT=$(dirname "$(dirname "$(which mysql)")") - export VT_MYSQL_ROOT - fi +# According to https://github.com/etcd-io/etcd/blob/a621d807f061e1dd635033a8d6bc261461429e27/Documentation/op-guide/supported-platform.md, +# currently, etcd is unstable on arm64, so ETCD_UNSUPPORTED_ARCH should be set. +if [ "$(uname -m)" == aarch64 ]; then + export ETCD_UNSUPPORTED_ARCH=arm64 fi - -PKG_CONFIG_PATH=$(prepend_path "$PKG_CONFIG_PATH" "$VTROOT/lib") -export PKG_CONFIG_PATH - -# Useful aliases. Remove if inconvenient. -alias gt='cd $GOTOP' -alias pt='cd $PYTOP' -alias vt='cd $VTTOP' diff --git a/doc/V3HighLevelDesign.md b/doc/V3HighLevelDesign.md index b0ba3268a7b..5becdb0d46b 100644 --- a/doc/V3HighLevelDesign.md +++ b/doc/V3HighLevelDesign.md @@ -782,7 +782,7 @@ Recapitulating what we’ve covered so far: Once we start allowing joins and subqueries, we have a whole bunch of table aliases and relationships to deal with. We have to contend with name clashes, self-joins, as well as scoping rules. In a way, the vschema has acted as a static symbol table so far. But that’s not going to be enough any more. -The core of the symbol table will contain a map whose key will be a table alias, and the elements will be [similar to the table in vschema](https://github.com/vitessio/vitess/blob/master/go/vt/vtgate/planbuilder/schema.go#L22). However, it will also contain a column list that will be built as the query is parsed. +The core of the symbol table will contain a map whose key will be a table alias, and the elements will be [similar to the table in vschema](https://github.com/vitessio/vitess/blob/0b3de7c4a2de8daec545f040639b55a835361685/go/vt/vtgate/vindexes/vschema.go#L82). However, it will also contain a column list that will be built as the query is parsed. ### A simple example diff --git a/doc/VitessQueues.md b/doc/VitessQueues.md index 20a2b951669..6b0d02823d8 100644 --- a/doc/VitessQueues.md +++ b/doc/VitessQueues.md @@ -79,7 +79,7 @@ capabilities, the usual horizontal sharding process can be used. Queue Tables are marked in the schema by a comment, in a similar way we detect Sequence Tables -[now](https://github.com/vitessio/vitess/blob/master/go/vt/tabletserver/table_info.go#L37). +[now](https://github.com/vitessio/vitess/blob/0b3de7c4a2de8daec545f040639b55a835361685/go/vt/vttablet/tabletserver/tabletserver.go#L138). When a tablet becomes a master, and there are Queue tables, it creates a QueueManager for each of them. diff --git a/docker/README.md b/docker/README.md index 0b0e47d19fe..8627204bf49 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,7 +5,7 @@ This file describes the purpose of the different images. **TL;DR:** Use the [vitess/lite](https://hub.docker.com/r/vitess/lite/) image for running Vitess. Our Kubernetes Tutorial uses it as well. -Instead of using the `latest` tag, you can pin it to a known stable version e.g. `v2.0`. +Instead of using the `latest` tag, you can pin it to a known stable version e.g. `v4.0`. ## Principles @@ -37,9 +37,7 @@ Our list of images can be grouped into: All these Vitess images include a specific MySQL/MariaDB version ("flavor"). * We provide Dockerfile files for multiple flavors (`Dockerfile.`). - * As of April 2017, the following flavors are supported: `mariadb`, `mysql56`, `mysql57`, `percona`(56), `percona57` * On Docker Hub we publish only images with MySQL 5.7 to minimize maintenance overhead and avoid confusion. - * If you need an image for a different flavor, it is very easy to build it yourself. See the [Custom Docker Build instructions](https://vitess.io/getting-started/docker-build/). If you are looking for a stable version of Vitess, use the **lite** image with a fixed version. If you are looking for the latest Vitess code in binary form, use the "latest" tag of the **base** image. @@ -50,11 +48,3 @@ If you are looking for a stable version of Vitess, use the **lite** image with a | **guestbook** | manual (updated with every Vitess release) | Vitess adaption of the Kubernetes guestbook example. Used to showcase sharding in Vitess. Dockerfile is located in [`examples/kubernetes/guestbook/`](https://github.com/vitessio/vitess/tree/master/examples/kubernetes/guestbook). | | **orchestrator** | manual | Binaries for [Orchestrator](https://github.com/github/orchestrator). It can be used with Vitess for automatic failovers. Currently not part of the Kubernetes Tutorial and only used in tests. | -### Internal Tools - -These images are used by the Vitess project for internal workflows and testing infrastructure and can be ignored by users. - -| Image | How (When) Updated | Description | -| --- | --- | --- | -| **publish-site** | manual | Contains [Jekyll](https://jekyllrb.com/) which we use to generate our [vitess.io](https://vitess.io) website from the Markdown files located in [doc/](https://github.com/vitessio/vitess/tree/master/doc). | -| **keytar** | manual | Keytar is a Vitess testing framework to run our Kubernetes cluster tests. Dockerfile is located in [`test/cluster/keytar/`](https://github.com/vitessio/vitess/tree/master/test/cluster/keytar). | diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 4b6a21beb49..6a9255b38e7 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -14,7 +14,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins libtool \ make \ openjdk-8-jdk \ - pkg-config \ python-crypto \ python-dev \ python-mysqldb \ @@ -31,26 +30,19 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && rm -rf /var/lib/apt/lists/* # Install Maven 3.1+ -RUN mkdir -p /vt/dist && \ - cd /vt/dist && \ +RUN mkdir -p /vt/src/vitess.io/vitess/dist && \ + cd /vt/src/vitess.io/vitess/dist && \ curl -sL --connect-timeout 10 --retry 3 \ http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && \ mv apache-maven-3.3.9 maven # Set up Vitess environment (equivalent to '. dev.env') -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV PYTOP $VTTOP/py -ENV VTDATAROOT $VTROOT/vtdataroot +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV VTPORTSTART 15000 ENV PYTHONPATH $VTROOT/dist/grpc/usr/local/lib/python2.7/site-packages:$VTROOT/dist/py-mock-1.0.1/lib/python2.7/site-packages:$VTROOT/py-vtdb:$VTROOT/dist/selenium/lib/python2.7/site-packages -ENV GOBIN $VTROOT/bin ENV PATH $VTROOT/bin:$VTROOT/dist/maven/bin:$VTROOT/dist/chromedriver:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib ENV USER vitess -ENV GO111MODULE on # Copy files needed for bootstrap COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/src/vitess.io/vitess/ @@ -71,6 +63,10 @@ RUN cd /vt/src/vitess.io/vitess && \ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot +# The docker lite images copy from the builder in /vt/bin +# Add compatibility to the previous layout for now +RUN su vitess -c "mkdir -p /vt/src/vitess.io/vitess/bin && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin" + # If the user doesn't specify a command, load a shell. CMD ["/bin/bash"] diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index 19d5c9973d0..e0f6106837f 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -22,6 +22,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MariaDB USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index c2828ddd25d..024fe6a80b3 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -11,6 +11,5 @@ RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24 # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MariaDB103 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index b07fddedb61..69c5f4dee0b 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -17,6 +17,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.s # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 2fb1f4e1aa5..76c76dc47b3 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -18,7 +18,5 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess - -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index ec53895165f..e6d00bc5245 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -17,7 +17,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.poo # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess - -ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index 6d13fa4dfb0..910d3be10b1 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -16,6 +16,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 6ed54c76923..54c8477ffb6 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -17,6 +17,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 1ce9c52103f..c5ce5d5ee48 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -32,6 +32,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r # Bootstrap Vitess WORKDIR /vt/src/vitess.io/vitess -ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile index ddafcd95af7..391c352e4f8 100644 --- a/docker/k8s/Dockerfile +++ b/docker/k8s/Dockerfile @@ -25,9 +25,8 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV VTDATAROOT /vtdataroot -ENV VTTOP /vt/src/vitess.io/vitess # Prepare directory structure. RUN mkdir -p /vt && \ @@ -50,13 +49,13 @@ COPY --from=base /vt/bin/vtworker /vt/bin/ COPY --from=base /vt/bin/vtbackup /vt/bin/ # copy web admin files -COPY --from=base $VTTOP/web /vt/web/ +COPY --from=base $VTROOT/web /vt/web/ # copy vitess config -COPY --from=base $VTTOP/config/init_db.sql /vt/config/ +COPY --from=base $VTROOT/config/init_db.sql /vt/config/ # my.cnf include files -COPY --from=base $VTTOP/config/mycnf /vt/config/mycnf +COPY --from=base $VTROOT/config/mycnf /vt/config/mycnf # add vitess user and add permissions RUN groupadd -r --gid 2000 vitess && useradd -r -g vitess --uid 1000 vitess && \ diff --git a/docker/k8s/vtctlclient/Dockerfile b/docker/k8s/vtctlclient/Dockerfile index 1b1b9be3bc3..0cf4956691d 100644 --- a/docker/k8s/vtctlclient/Dockerfile +++ b/docker/k8s/vtctlclient/Dockerfile @@ -1,11 +1,11 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,5 +28,5 @@ COPY --from=k8s /vt/bin/vtctlclient /usr/bin/ # add vitess user/group and add permissions RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess - + CMD ["/usr/bin/vtctlclient"] diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index 268a68fffce..a63321a42f5 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -40,6 +40,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins gnupg dirmngr ca-certificates wget libdbd-mysql-perl rsync libaio1 libatomic1 libcurl3 libev4 \ && for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done \ && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' > /etc/apt/sources.list.d/mysql.list \ + && for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done \ + && echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \ + { \ + echo debconf debconf/frontend select Noninteractive; \ + echo percona-server-server-5.7 percona-server-server/root_password password 'unused'; \ + echo percona-server-server-5.7 percona-server-server/root_password_again password 'unused'; \ + } | debconf-set-selections \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive \ apt-get install -y --no-install-recommends \ @@ -47,23 +54,17 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins libmysqlclient20 \ mysql-client \ mysql-server \ - && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \ - && dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \ - && rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \ + libjemalloc1 \ + libtcmalloc-minimal4 \ + percona-xtrabackup-24 \ && rm -rf /var/lib/apt/lists/* \ - && groupadd -r vitess && useradd -r -g vitess vitess + && groupadd -r vitess && useradd -r -g vitess vitess \ + && rm -rf /var/lib/mysql/ # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index f3d282163a9..b8fc9467f9a 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -22,16 +22,9 @@ RUN echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/reposito apk add --no-cache mariadb@edge mariadb-client@edge bzip2 bash # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB103 # Create vitess user RUN addgroup -S vitess && adduser -S -G vitess vitess && mkdir -p /vt diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index cea94d615e5..77eddfd8003 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -33,16 +33,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 4ff440d3d86..a1d37b4c42c 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -32,16 +32,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB103 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index f3d6b3dcb7a..bb434808372 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -36,16 +36,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 78318e5c652..809a79f9151 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -36,16 +36,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index fb71b6f6b56..0fade69bd35 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -36,16 +36,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index e8d127dc56f..0698f4eb583 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -38,16 +38,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 30472af2a60..d52fbaa583a 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -39,16 +39,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 31182d277d6..5d872499cf2 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -41,16 +41,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess -ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV VTDATAROOT $VTROOT/vtdataroot -ENV GOBIN $VTROOT/bin -ENV GOPATH $VTROOT +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH -ENV VT_MYSQL_ROOT /usr -ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/packaging/Dockerfile b/docker/packaging/Dockerfile deleted file mode 100644 index 576b8ef0e63..00000000000 --- a/docker/packaging/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM vitess/base - -USER root - -# Install gem and use gem to install fpm -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential \ - ruby-dev \ - rubygems \ - rpm \ - && rm -rf /var/lib/apt/lists/* \ - && gem install --no-ri --no-rdoc fpm - -RUN mkdir /vt/packaging - -COPY docker/packaging/* /vt/packaging/ - -RUN chown -R vitess:vitess /vt/packaging - -USER vitess - -ENTRYPOINT ["/bin/bash", "/vt/packaging/package_vitess.sh"] diff --git a/docker/packaging/package_vitess.sh b/docker/packaging/package_vitess.sh deleted file mode 100755 index 609fd1447d5..00000000000 --- a/docker/packaging/package_vitess.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -z "${VERSION}" ]; then - echo "Set the env var VERSION with the release version" - exit 1 -fi - -set -eu - -PREFIX=${PREFIX:-/usr} - -inputs_file="/vt/packaging/inputs" -cat <> "${inputs_file}" -/vt/bin/mysqlctld=${PREFIX}/bin/mysqlctld -/vt/bin/vtbackup=${PREFIX}/bin/vtbackup -/vt/bin/vtctl=${PREFIX}/bin/vtctl -/vt/bin/vtctlclient=${PREFIX}/bin/vtctlclient -/vt/bin/vtctld=${PREFIX}/bin/vtctld -/vt/bin/vtgate=${PREFIX}/bin/vtgate -/vt/bin/vttablet=${PREFIX}/bin/vttablet -/vt/bin/vtworker=${PREFIX}/bin/vtworker -/vt/src/vitess.io/vitess/config/=/etc/vitess -/vt/src/vitess.io/vitess/web/vtctld2/app=${PREFIX}/lib/vitess/web/vtcld2 -/vt/src/vitess.io/vitess/web/vtctld=${PREFIX}/lib/vitess/web -/vt/src/vitess.io/vitess/examples/local/=${PREFIX}/share/vitess/examples -EOF - -description='A database clustering system for horizontal scaling of MySQL - -Vitess is a database solution for deploying, scaling and managing large -clusters of MySQL instances. It’s architected to run as effectively in a public -or private cloud architecture as it does on dedicated hardware. It combines and -extends many important MySQL features with the scalability of a NoSQL database.' - -exec /usr/local/bin/fpm \ - --force \ - --input-type dir \ - --name vitess \ - --version "${VERSION}" \ - --url "https://vitess.io/" \ - --description "${description}" \ - --license "Apache License - Version 2.0, January 2004" \ - --inputs "${inputs_file}" \ - --config-files "/etc/vitess" \ - --directories "${PREFIX}/lib/vitess" \ - --before-install "/vt/packaging/preinstall.sh" \ - "${@}" diff --git a/docker/publish-site/Dockerfile b/docker/publish-site/Dockerfile deleted file mode 100644 index e0a1946996f..00000000000 --- a/docker/publish-site/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This image should be built with $VTTOP as the context dir. -# For example: -# vitess$ docker build -f docker/publish-site/Dockerfile -t vitess/publish-site . -FROM ruby:2.3 - -# Install apt dependencies. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Install ruby dependencies. -COPY vitess.io/Gemfile /vitess.io/Gemfile -RUN cd /vitess.io && \ - gem install bundler && \ - bundle install - -# Expose port for preview-site.sh. -EXPOSE 4000 - diff --git a/docker/test/run.sh b/docker/test/run.sh index 147663d028b..df0c6a20f82 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -160,37 +160,32 @@ case "$mode" in "create_cache") echo "Creating cache image $cache_image ..." ;; esac -# Construct "cp" command to copy the source code. -# -# Copy the full source tree except: -# - vendor -# That's because these directories are already part of the image. -# -# Note that we're using the Bash extended Glob support "!(vendor)" on -# purpose here to minimize the size of the cache image: With this trick, -# we do not move or overwrite the existing files while copying the other -# directories. Therefore, the existing files do not count as changed and will -# not be part of the new Docker layer of the cache image. -copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ." -# Copy the .git directory because travis/check_make_proto.sh needs a working -# Git repository. -copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") - -# Enable gomodules -run_bootstrap_cmd="export GO111MODULE=on" -# Copy bootstrap.sh if it changed -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") -# run bootstrap.sh if necessary -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") -copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") - -# Construct the command we will actually run. -# -# Uncomment the next line if you need to debug "bashcmd". -#bashcmd="set -x" +bashcmd="" + if [[ -z "$existing_cache_image" ]]; then - bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd") + + # Construct "cp" command to copy the source code. + bashcmd=$(append_cmd "$bashcmd" "cp -R /tmp/src/!(vtdataroot|dist|bin|lib|vthook|py-vtdb) . && cp -R /tmp/src/.git .") + fi + +# Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence. +bashcmd=$(append_cmd "$bashcmd" "export VTROOT=/vt/src/vitess.io/vitess") +bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") + +bashcmd=$(append_cmd "$bashcmd" "mkdir -p dist; mkdir -p bin; mkdir -p lib; mkdir -p vthook") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/dist; ln -s /vt/src/vitess.io/vitess/dist /vt/dist") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/bin; ln -s /vt/src/vitess.io/vitess/bin /vt/bin") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/lib /vt/lib") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") + +# Maven was setup in /vt/dist, may need to reinstall it. +bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'; if [[ ! \$(command -v mvn) ]]; then echo 'install maven'; curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven; fi; echo 'mvn check done'") + +# Run bootstrap every time now +bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") + # At last, append the user's command. bashcmd=$(append_cmd "$bashcmd" "$cmd") diff --git a/examples/compose/README.md b/examples/compose/README.md index eb801a8fcbe..7d4e4a2bc07 100644 --- a/examples/compose/README.md +++ b/examples/compose/README.md @@ -51,7 +51,7 @@ Flags available: go run vtcompose/vtcompose.go -keyspaceData="test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace;lookup_keyspace:1:1:create_tokens_token_lookup.sql,create_messages_message_lookup.sql" ``` * **externalDbData** - Specifies which databases/keyspaces are external and provides data along with it to connect to the external db. - List of `,,,,,` seperated by ';'. + List of `,,,,,` separated by ';'. When using this, make sure to have the external_db_name/keyspace in the `keyspaceData` flag with no schema_file_names specified. ``` go run vtcompose/vtcompose.go -keyspaces="test:0:2::" -externalDbData="test:192.68.99.101:3306:admin:pass:CHARACTER SET utf8 COLLATE utf8_general_ci" @@ -95,13 +95,18 @@ vitess/examples/compose$ ./client.sh ### Connect to vgate and run queries vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line. -You can also use the `./lmysql.sh` helper script. ``` vitess/examples/compose$ mysql --port=15306 --host=127.0.0.1 -vitess/examples/compose$ ./lmysql.sh --port=15306 --host=127.0.0.1 ``` **Note that you may need to replace `127.0.0.1` with `docker ip` or `docker-machine ip`** +You can also use the `./lmysql.sh` helper script. +``` +vitess/examples/compose$ ./lmysql.sh --port=15306 --host= +``` + +where `` is `docker-machine ip` or external docker host ip addr + ### Play around with vtctl commands ``` @@ -152,7 +157,6 @@ DB_CHARSET=CHARACTER SET utf8 COLLATE utf8_general_ci Ensure you have log bin enabled on your external database. You may add the following configs to your conf.d directory and reload mysqld on your server ``` -vitess/config/mycnf/master_mysql56.cnf vitess/config/mycnf/rbr.cnf ``` @@ -258,4 +262,4 @@ vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "t ``` This has since been fixed by -https://github.com/vitessio/vitess/pull/4868 & https://github.com/vitessio/vitess/pull/5010 \ No newline at end of file +https://github.com/vitessio/vitess/pull/4868 & https://github.com/vitessio/vitess/pull/5010 diff --git a/examples/compose/docker-compose.beginners.yml b/examples/compose/docker-compose.beginners.yml index 4f23b688f1a..67294b90c69 100644 --- a/examples/compose/docker-compose.beginners.yml +++ b/examples/compose/docker-compose.beginners.yml @@ -37,8 +37,8 @@ services: command: ["sh", "-c", " $$VTROOT/bin/vtctld \ $TOPOLOGY_FLAGS \ -cell $CELL \ - -web_dir $$VTTOP/web/vtctld \ - -web_dir2 $$VTTOP/web/vtctld2/app \ + -web_dir $$VTROOT/web/vtctld \ + -web_dir2 $$VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/examples/compose/external_db/docker-compose.yml b/examples/compose/external_db/docker-compose.yml index 5b3b28f1f9e..b0b1e58f9fd 100644 --- a/examples/compose/external_db/docker-compose.yml +++ b/examples/compose/external_db/docker-compose.yml @@ -17,7 +17,6 @@ services: volumes: - vol-db:/var/lib/mysql - ./mysql/:/docker-entrypoint-initdb.d/ - - ./mysql/master_mysql56.cnf:/etc/mysql/conf.d/master_mysql56.cnf - ./mysql/query.log:/var/log/mysql/query.log - ./mysql/slow.log:/var/log/mysql/slow.log healthcheck: diff --git a/examples/compose/lmysql.sh b/examples/compose/lmysql.sh old mode 100644 new mode 100755 diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index b88099f3a6c..d4825214adc 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -48,7 +48,7 @@ var ( mySqlPort = flag.String("mySqlPort", "15306", "mySql port to be used") cell = flag.String("cell", "test", "Vitess Cell name") keyspaceData = flag.String("keyspaceData", "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") - externalDbData = flag.String("externalDbData", "", "List of Data corresponding to external DBs. List of ,,,,, seperated by ';'") + externalDbData = flag.String("externalDbData", "", "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") ) type keyspaceInfo struct { @@ -466,8 +466,8 @@ func generateVtctld() string { command: ["sh", "-c", " $$VTROOT/bin/vtctld \ %[3]s \ -cell %[4]s \ - -web_dir $$VTTOP/web/vtctld \ - -web_dir2 $$VTTOP/web/vtctld2/app \ + -web_dir $$VTROOT/web/vtctld \ + -web_dir2 $$VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/examples/compose/vttablet-up.sh b/examples/compose/vttablet-up.sh index 00369ecdd22..78623e2fb1c 100755 --- a/examples/compose/vttablet-up.sh +++ b/examples/compose/vttablet-up.sh @@ -68,9 +68,6 @@ if [ $tablet_role != "master" ]; then echo "CREATE DATABASE IF NOT EXISTS $db_name;" >> $init_db_sql_file fi fi -# Enforce Row Based Replication -export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf -export EXTRA_MY_CNF=$EXTRA_MY_CNF:$VTROOT/config/mycnf/master_mysql56.cnf mkdir -p $VTDATAROOT/backups @@ -182,4 +179,4 @@ exec $VTROOT/bin/vttablet \ -backup_storage_implementation file \ -file_backup_storage_root $VTDATAROOT/backups \ -queryserver-config-schema-reload-time 60 \ - $external_db_args \ No newline at end of file + $external_db_args diff --git a/examples/demo/run.py b/examples/demo/run.py index 45eb8267ce2..5f8916bb176 100755 --- a/examples/demo/run.py +++ b/examples/demo/run.py @@ -51,13 +51,14 @@ def start_vitess(): keyspace = topology.keyspaces.add(name='lookup') keyspace.shards.add(name='0') - vttop = os.environ['VTTOP'] - args = [os.path.join(vttop, 'py/vttest/run_local_database.py'), + vtroot = os.environ['VTROOT'] + args = [os.path.join(vtroot, 'py/vttest/run_local_database.py'), '--port', '12345', '--proto_topo', text_format.MessageToString(topology, as_one_line=True), '--web_dir', os.path.join(vttop, 'web/vtctld'), - '--schema_dir', os.path.join(vttop, 'examples/demo/schema')] + '--schema_dir', os.path.join(vttop, 'examples/demo/schema'), + '--mysql_server_bind_address', '0.0.0.0'] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) # This load will make us wait for vitess to come up. diff --git a/examples/helm/101_initial_cluster.yaml b/examples/helm/101_initial_cluster.yaml index 79d9bb972ea..aab85cee688 100644 --- a/examples/helm/101_initial_cluster.yaml +++ b/examples/helm/101_initial_cluster.yaml @@ -64,7 +64,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/201_customer_keyspace.yaml b/examples/helm/201_customer_keyspace.yaml index 3490ac2e106..c343abb8d97 100644 --- a/examples/helm/201_customer_keyspace.yaml +++ b/examples/helm/201_customer_keyspace.yaml @@ -40,7 +40,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/202_customer_tablets.yaml b/examples/helm/202_customer_tablets.yaml index f24c929e4e1..2a13c3940f2 100644 --- a/examples/helm/202_customer_tablets.yaml +++ b/examples/helm/202_customer_tablets.yaml @@ -65,7 +65,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/203_vertical_split.yaml b/examples/helm/203_vertical_split.yaml index 677f2bbd9d5..1bd2de6fd8f 100644 --- a/examples/helm/203_vertical_split.yaml +++ b/examples/helm/203_vertical_split.yaml @@ -51,7 +51,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/204_vertical_migrate_replicas.yaml b/examples/helm/204_vertical_migrate_replicas.yaml index 360c30020e9..aad9a9b155c 100644 --- a/examples/helm/204_vertical_migrate_replicas.yaml +++ b/examples/helm/204_vertical_migrate_replicas.yaml @@ -53,7 +53,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/205_vertical_migrate_master.yaml b/examples/helm/205_vertical_migrate_master.yaml index 3476c6709be..21ecc757762 100644 --- a/examples/helm/205_vertical_migrate_master.yaml +++ b/examples/helm/205_vertical_migrate_master.yaml @@ -50,7 +50,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/206_clean_commerce.yaml b/examples/helm/206_clean_commerce.yaml index 57304dea4f8..11883808f52 100644 --- a/examples/helm/206_clean_commerce.yaml +++ b/examples/helm/206_clean_commerce.yaml @@ -60,7 +60,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/301_customer_sharded.yaml b/examples/helm/301_customer_sharded.yaml index 34ef10ca90f..d20497d6b93 100644 --- a/examples/helm/301_customer_sharded.yaml +++ b/examples/helm/301_customer_sharded.yaml @@ -104,7 +104,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/302_new_shards.yaml b/examples/helm/302_new_shards.yaml index 9ed6098cd72..658281899ad 100644 --- a/examples/helm/302_new_shards.yaml +++ b/examples/helm/302_new_shards.yaml @@ -65,7 +65,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/303_horizontal_split.yaml b/examples/helm/303_horizontal_split.yaml index de6b2c527b4..7415384db45 100644 --- a/examples/helm/303_horizontal_split.yaml +++ b/examples/helm/303_horizontal_split.yaml @@ -67,7 +67,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/304_migrate_replicas.yaml b/examples/helm/304_migrate_replicas.yaml index bb5b35efa7a..950b1e8ce38 100644 --- a/examples/helm/304_migrate_replicas.yaml +++ b/examples/helm/304_migrate_replicas.yaml @@ -69,7 +69,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/305_migrate_master.yaml b/examples/helm/305_migrate_master.yaml index 8b325b2bc05..ae3dd170105 100644 --- a/examples/helm/305_migrate_master.yaml +++ b/examples/helm/305_migrate_master.yaml @@ -66,7 +66,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/306_down_shard_0.yaml b/examples/helm/306_down_shard_0.yaml index 6b3fb1b3a01..d34ac318f65 100644 --- a/examples/helm/306_down_shard_0.yaml +++ b/examples/helm/306_down_shard_0.yaml @@ -53,7 +53,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/307_delete_shard_0.yaml b/examples/helm/307_delete_shard_0.yaml index f24e3410619..ccfad668d30 100644 --- a/examples/helm/307_delete_shard_0.yaml +++ b/examples/helm/307_delete_shard_0.yaml @@ -58,7 +58,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/308_final.yaml b/examples/helm/308_final.yaml index f24e3410619..ccfad668d30 100644 --- a/examples/helm/308_final.yaml +++ b/examples/helm/308_final.yaml @@ -58,7 +58,7 @@ vtgate: resources: vttablet: - mysqlSize: "test" + mysqlSize: "prod" resources: mysqlResources: diff --git a/examples/helm/kvtctld.sh b/examples/helm/kvtctld.sh old mode 100755 new mode 100644 index 45f9c796299..2499e706301 --- a/examples/helm/kvtctld.sh +++ b/examples/helm/kvtctld.sh @@ -1,13 +1,13 @@ #!/bin/bash # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/examples/kubernetes/etcd-down.sh b/examples/kubernetes/etcd-down.sh old mode 100755 new mode 100644 index ecc912b63de..3aef6636bd7 --- a/examples/kubernetes/etcd-down.sh +++ b/examples/kubernetes/etcd-down.sh @@ -1,13 +1,13 @@ #!/bin/bash # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/examples/kubernetes/guestbook/extract.sh b/examples/kubernetes/guestbook/extract.sh index e029df3fc77..04499c87397 100644 --- a/examples/kubernetes/guestbook/extract.sh +++ b/examples/kubernetes/guestbook/extract.sh @@ -17,7 +17,7 @@ set -e # Collect all the local Python libs we need. mkdir -p /out/pkg/py-vtdb -cp -R $VTTOP/py/* /out/pkg/py-vtdb/ +cp -R $VTROOT/py/* /out/pkg/py-vtdb/ cp -R /usr/local/lib/python2.7/dist-packages /out/pkg/ cp -R /vt/dist/py-* /out/pkg/ diff --git a/examples/kubernetes/vtctld-controller-template.yaml b/examples/kubernetes/vtctld-controller-template.yaml index 2ca6f6d80c1..4e28aa8bfc4 100644 --- a/examples/kubernetes/vtctld-controller-template.yaml +++ b/examples/kubernetes/vtctld-controller-template.yaml @@ -41,8 +41,8 @@ spec: chown -R vitess /vt && su -p -c "/vt/bin/vtctld -cell {{cell}} - -web_dir $VTTOP/web/vtctld - -web_dir2 $VTTOP/web/vtctld2/app + -web_dir $VTROOT/web/vtctld + -web_dir2 $VTROOT/web/vtctld2/app -workflow_manager_init -workflow_manager_use_election -log_dir $VTDATAROOT/tmp diff --git a/examples/kubernetes/vttablet-pod-benchmarking-template.yaml b/examples/kubernetes/vttablet-pod-benchmarking-template.yaml index 8f24da64f54..f1cc59e7101 100644 --- a/examples/kubernetes/vttablet-pod-benchmarking-template.yaml +++ b/examples/kubernetes/vttablet-pod-benchmarking-template.yaml @@ -87,9 +87,6 @@ spec: -tablet_uid {{uid}} -socket_file $VTDATAROOT/mysqlctl.sock -init_db_sql_file $VTROOT/config/init_db.sql" vitess - env: - - name: EXTRA_MY_CNF - value: /vt/config/mycnf/benchmark.cnf:/vt/config/mycnf/master_mysql56.cnf volumes: - name: syslog hostPath: {path: /dev/log} diff --git a/examples/kubernetes/vttablet-pod-template.yaml b/examples/kubernetes/vttablet-pod-template.yaml index 52e19aad80f..6be80409cae 100644 --- a/examples/kubernetes/vttablet-pod-template.yaml +++ b/examples/kubernetes/vttablet-pod-template.yaml @@ -69,9 +69,6 @@ spec: -orc_api_url http://orchestrator/api -orc_discover_interval 5m -restore_from_backup {{backup_flags}}" vitess - env: - - name: EXTRA_MY_CNF - value: /vt/config/mycnf/master_mysql56.cnf - name: mysql image: {{vitess_image}} volumeMounts: @@ -96,9 +93,6 @@ spec: -tablet_uid {{uid}} -socket_file $VTDATAROOT/mysqlctl.sock -init_db_sql_file $VTROOT/config/init_db.sql" vitess - env: - - name: EXTRA_MY_CNF - value: /vt/config/mycnf/master_mysql56.cnf volumes: - name: syslog hostPath: {path: /dev/log} diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh index 9f018da6b55..1a484d98d5d 100755 --- a/examples/local/101_initial_cluster.sh +++ b/examples/local/101_initial_cluster.sh @@ -21,11 +21,7 @@ set -e # shellcheck disable=SC2128 script_root=$(dirname "${BASH_SOURCE}") - -if [[ $EUID -eq 0 ]]; then - echo "This script refuses to be run as root. Please switch to a regular user." - exit 1 -fi +source "${script_root}/env.sh" # start topo server if [ "${TOPO}" = "zk2" ]; then diff --git a/examples/local/env.sh b/examples/local/env.sh index ef9a0c8538a..d36a1cc3ffe 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -16,29 +16,32 @@ hostname=`hostname -f` vtctld_web_port=15000 +export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" -# Set up environment. -export VTTOP=${VTTOP-$VTROOT/src/vitess.io/vitess} +function fail() { + echo "ERROR: $1" + exit 1 +} -# Try to find mysqld on PATH. -if [ -z "$VT_MYSQL_ROOT" ]; then - mysql_path=`which mysqld` - if [ -z "$mysql_path" ]; then - echo "Can't guess location of mysqld. Please set VT_MYSQL_ROOT manually." - exit 1 - fi - export VT_MYSQL_ROOT=$(dirname `dirname $mysql_path`) +if [[ $EUID -eq 0 ]]; then + fail "This script refuses to be run as root. Please switch to a regular user." fi -# Previously the file specified MYSQL_FLAVOR -# it is now autodetected +# mysqld might be in /usr/sbin which will not be in the default PATH +PATH="/usr/sbin:$PATH" +for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do + command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions." +done; + +if [ -z "$VTROOT" ]; then + fail "VTROOT is not set. See https://vitess.io/docs/get-started/local/ for install instructions." +fi if [ "${TOPO}" = "zk2" ]; then # Each ZooKeeper server needs a list of all servers in the quorum. # Since we're running them all locally, we need to give them unique ports. # In a real deployment, these should be on different machines, and their # respective hostnames should be given. - echo "enter zk2 env" zkcfg=(\ "1@$hostname:28881:38881:21811" \ "2@$hostname:28882:38882:21812" \ @@ -56,16 +59,7 @@ if [ "${TOPO}" = "zk2" ]; then mkdir -p $VTDATAROOT/tmp else - echo "enter etcd2 env" - - case $(uname) in - Linux) etcd_platform=linux;; - Darwin) etcd_platform=darwin;; - esac - ETCD_SERVER="localhost:2379" - ETCD_VERSION=$(cat "${VTROOT}/dist/etcd/.installed_version") - ETCD_BINDIR="${VTROOT}/dist/etcd/etcd-${ETCD_VERSION}-${etcd_platform}-amd64/" TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global" mkdir -p "${VTDATAROOT}/tmp" diff --git a/examples/local/etcd-down.sh b/examples/local/etcd-down.sh index 9d05f649c44..cb2d5b1023f 100755 --- a/examples/local/etcd-down.sh +++ b/examples/local/etcd-down.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is an example script that stops the ZooKeeper servers started by zk-up.sh. +# This is an example script that stops the etcd servers started by etcd-up.sh. set -e diff --git a/examples/local/etcd-up.sh b/examples/local/etcd-up.sh index 05d25ab0e28..36cd4269565 100755 --- a/examples/local/etcd-up.sh +++ b/examples/local/etcd-up.sh @@ -20,22 +20,26 @@ set -e cell=${CELL:-'test'} script_root=$(dirname "${BASH_SOURCE[0]}") +export ETCDCTL_API=2 # shellcheck source=./env.sh # shellcheck disable=SC1091 source "${script_root}/env.sh" -${ETCD_BINDIR}/etcd --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & +# Check that etcd is not already running +curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." + +etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & PID=$! echo $PID > "${VTDATAROOT}/tmp/etcd.pid" sleep 5 echo "add /vitess/global" -${ETCD_BINDIR}/etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global & +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global & echo "add /vitess/$cell" -${ETCD_BINDIR}/etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & # And also add the CellInfo description for the cell. # If the node already exists, it's fine, means we used existing data. diff --git a/examples/local/vtctld-up.sh b/examples/local/vtctld-up.sh index dbd16a8de66..128152b837d 100755 --- a/examples/local/vtctld-up.sh +++ b/examples/local/vtctld-up.sh @@ -36,8 +36,8 @@ echo "Starting vtctld..." $VTROOT/bin/vtctld \ $TOPOLOGY_FLAGS \ -cell $cell \ - -web_dir $VTTOP/web/vtctld \ - -web_dir2 $VTTOP/web/vtctld2/app \ + -web_dir $VTROOT/web/vtctld \ + -web_dir2 $VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/go.mod b/go.mod index ae3b4d9aee4..44d8c461dd1 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a github.com/boltdb/bolt v1.3.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/coreos/etcd v0.0.0-20170626015032-703663d1f6ed @@ -24,6 +25,7 @@ require ( github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.0-20170215233205-553a64147049 + github.com/google/btree v1.0.0 // indirect github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 @@ -48,10 +50,13 @@ require ( github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing/opentracing-go v1.1.0 github.com/pborman/uuid v0.0.0-20160824210600-b984ec7fa9ff + github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.1.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect github.com/prometheus/common v0.7.0 // indirect diff --git a/go.sum b/go.sum index ff202019b03..df4e4fb847c 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go index 05717c42f87..944e593a5e1 100644 --- a/go/cmd/vtbackup/vtbackup.go +++ b/go/cmd/vtbackup/vtbackup.go @@ -379,12 +379,17 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back } } + // Stop replication and see where we are. + if err := mysqld.StopSlave(nil); err != nil { + return fmt.Errorf("can't stop replication: %v", err) + } + // Did we make any progress? status, err := mysqld.SlaveStatus() if err != nil { return fmt.Errorf("can't get replication status: %v", err) } - log.Infof("Replication caught up to at least %v", status.Position) + log.Infof("Replication caught up to %v", status.Position) if !status.Position.AtLeast(masterPos) && status.Position.Equal(restorePos) { return fmt.Errorf("not taking backup: replication did not make any progress from restore point: %v", restorePos) } diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 837bc49fc89..c21f22ca388 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -61,7 +61,7 @@ func main() { servenv.Init() if *tabletPath == "" { - log.Exit("tabletPath required") + log.Exit("-tablet-path required") } tabletAlias, err := topoproto.ParseTabletAlias(*tabletPath) if err != nil { diff --git a/go/cmd/zk/zkcmd.go b/go/cmd/zk/zkcmd.go index 67a34773573..1b01f19aaac 100644 --- a/go/cmd/zk/zkcmd.go +++ b/go/cmd/zk/zkcmd.go @@ -177,7 +177,7 @@ func main() { func fixZkPath(zkPath string) string { if zkPath != "/" { - zkPath = strings.TrimRight(zkPath, "/") + zkPath = strings.TrimSuffix(zkPath, "/") } return path.Clean(zkPath) } diff --git a/go/mysql/binlog_event_filepos.go b/go/mysql/binlog_event_filepos.go new file mode 100644 index 00000000000..9b2b6e1cef8 --- /dev/null +++ b/go/mysql/binlog_event_filepos.go @@ -0,0 +1,246 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "encoding/binary" + "fmt" + "strconv" +) + +// filePosBinlogEvent wraps a raw packet buffer and provides methods to examine +// it by implementing BinlogEvent. Some methods are pulled in from binlogEvent. +type filePosBinlogEvent struct { + binlogEvent +} + +func (*filePosBinlogEvent) GTID(BinlogFormat) (GTID, bool, error) { + return nil, false, nil +} + +func (*filePosBinlogEvent) IsGTID() bool { + return false +} + +func (*filePosBinlogEvent) PreviousGTIDs(BinlogFormat) (Position, error) { + return Position{}, fmt.Errorf("filePos should not provide PREVIOUS_GTIDS_EVENT events") +} + +// StripChecksum implements BinlogEvent.StripChecksum(). +func (ev *filePosBinlogEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { + switch f.ChecksumAlgorithm { + case BinlogChecksumAlgOff, BinlogChecksumAlgUndef: + // There is no checksum. + return ev, nil, nil + default: + // Checksum is the last 4 bytes of the event buffer. + data := ev.Bytes() + length := len(data) + checksum := data[length-4:] + data = data[:length-4] + return &filePosBinlogEvent{binlogEvent: binlogEvent(data)}, checksum, nil + } +} + +// nextPosition returns the next file position of the binlog. +// If no information is available, it returns 0. +func (ev *filePosBinlogEvent) nextPosition(f BinlogFormat) int { + if f.HeaderLength <= 13 { + // Dead code. This is just a failsafe. + return 0 + } + return int(binary.LittleEndian.Uint32(ev.Bytes()[13:17])) +} + +// rotate implements BinlogEvent.Rotate(). +// +// Expected format (L = total length of event data): +// # bytes field +// 8 position +// 8:L file +func (ev *filePosBinlogEvent) rotate(f BinlogFormat) (int, string) { + data := ev.Bytes()[f.HeaderLength:] + pos := binary.LittleEndian.Uint64(data[0:8]) + file := data[8:] + return int(pos), string(file) +} + +//---------------------------------------------------------------------------- + +// filePosQueryEvent is a fake begin event. +type filePosQueryEvent struct { + query string + filePosFakeEvent +} + +func newFilePosQueryEvent(query string, ts uint32) filePosQueryEvent { + return filePosQueryEvent{ + query: query, + filePosFakeEvent: filePosFakeEvent{ + timestamp: ts, + }, + } +} + +func (ev filePosQueryEvent) IsQuery() bool { + return true +} + +func (ev filePosQueryEvent) Query(BinlogFormat) (Query, error) { + return Query{ + SQL: ev.query, + }, nil +} + +func (ev filePosQueryEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { + return ev, nil, nil +} + +//---------------------------------------------------------------------------- + +// filePosFakeEvent is the base class for fake events. +type filePosFakeEvent struct { + timestamp uint32 +} + +func (ev filePosFakeEvent) IsValid() bool { + return true +} + +func (ev filePosFakeEvent) IsFormatDescription() bool { + return false +} + +func (ev filePosFakeEvent) IsQuery() bool { + return false +} + +func (ev filePosFakeEvent) IsXID() bool { + return false +} + +func (ev filePosFakeEvent) IsGTID() bool { + return false +} + +func (ev filePosFakeEvent) IsRotate() bool { + return false +} + +func (ev filePosFakeEvent) IsIntVar() bool { + return false +} + +func (ev filePosFakeEvent) IsRand() bool { + return false +} + +func (ev filePosFakeEvent) IsPreviousGTIDs() bool { + return false +} + +func (ev filePosFakeEvent) IsTableMap() bool { + return false +} + +func (ev filePosFakeEvent) IsWriteRows() bool { + return false +} + +func (ev filePosFakeEvent) IsUpdateRows() bool { + return false +} + +func (ev filePosFakeEvent) IsDeleteRows() bool { + return false +} + +func (ev filePosFakeEvent) Timestamp() uint32 { + return ev.timestamp +} + +func (ev filePosFakeEvent) Format() (BinlogFormat, error) { + return BinlogFormat{}, nil +} + +func (ev filePosFakeEvent) GTID(BinlogFormat) (GTID, bool, error) { + return nil, false, nil +} + +func (ev filePosFakeEvent) Query(BinlogFormat) (Query, error) { + return Query{}, nil +} + +func (ev filePosFakeEvent) IntVar(BinlogFormat) (byte, uint64, error) { + return 0, 0, nil +} + +func (ev filePosFakeEvent) Rand(BinlogFormat) (uint64, uint64, error) { + return 0, 0, nil +} + +func (ev filePosFakeEvent) PreviousGTIDs(BinlogFormat) (Position, error) { + return Position{}, nil +} + +func (ev filePosFakeEvent) TableID(BinlogFormat) uint64 { + return 0 +} + +func (ev filePosFakeEvent) TableMap(BinlogFormat) (*TableMap, error) { + return nil, nil +} + +func (ev filePosFakeEvent) Rows(BinlogFormat, *TableMap) (Rows, error) { + return Rows{}, nil +} + +func (ev filePosFakeEvent) IsPseudo() bool { + return false +} + +//---------------------------------------------------------------------------- + +// filePosGTIDEvent is a fake GTID event for filePos. +type filePosGTIDEvent struct { + filePosFakeEvent + gtid filePosGTID +} + +func newFilePosGTIDEvent(file string, pos int, timestamp uint32) filePosGTIDEvent { + return filePosGTIDEvent{ + filePosFakeEvent: filePosFakeEvent{ + timestamp: timestamp, + }, + gtid: filePosGTID{ + file: file, + pos: strconv.Itoa(pos), + }, + } +} + +func (ev filePosGTIDEvent) IsGTID() bool { + return true +} + +func (ev filePosGTIDEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { + return ev, nil, nil +} + +func (ev filePosGTIDEvent) GTID(BinlogFormat) (GTID, bool, error) { + return ev.gtid, false, nil +} diff --git a/go/mysql/client.go b/go/mysql/client.go index d91a6fbbbce..d679de6d733 100644 --- a/go/mysql/client.go +++ b/go/mysql/client.go @@ -226,6 +226,7 @@ func (c *Conn) clientHandshake(characterSet uint8, params *ConnParams) error { if err != nil { return err } + c.fillFlavor(params) // Sanity check. if capabilities&CapabilityClientProtocol41 == 0 { @@ -392,7 +393,6 @@ func (c *Conn) parseInitialHandshakePacket(data []byte) (uint32, []byte, error) if !ok { return 0, nil, NewSQLError(CRMalformedPacket, SSUnknownSQLState, "parseInitialHandshakePacket: packet has no server version") } - c.fillFlavor() // Read the connection id. c.ConnectionID, pos, ok = readUint32(data, pos) diff --git a/go/mysql/conn.go b/go/mysql/conn.go index 0d51867bd06..64161fe6ee1 100644 --- a/go/mysql/conn.go +++ b/go/mysql/conn.go @@ -897,12 +897,9 @@ func (c *Conn) handleNextCommand(handler Handler) error { if stmtID != uint32(0) { defer func() { + // Allocate a new bindvar map every time since VTGate.Execute() mutates it. prepare := c.PrepareData[stmtID] - if prepare.BindVars != nil { - for k := range prepare.BindVars { - prepare.BindVars[k] = nil - } - } + prepare.BindVars = make(map[string]*querypb.BindVariable, prepare.ParamsCount) }() } diff --git a/go/mysql/conn_params.go b/go/mysql/conn_params.go index 1bd772f0838..d3956346612 100644 --- a/go/mysql/conn_params.go +++ b/go/mysql/conn_params.go @@ -26,6 +26,7 @@ type ConnParams struct { UnixSocket string `json:"unix_socket"` Charset string `json:"charset"` Flags uint64 `json:"flags"` + Flavor string `json:"flavor,omitempty"` // The following SSL flags are only used when flags |= 2048 // is set (CapabilityClientSSL). diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go index df9a49017fe..3c086c22637 100644 --- a/go/mysql/fakesqldb/server.go +++ b/go/mysql/fakesqldb/server.go @@ -205,6 +205,7 @@ func (db *DB) Close() { db.listener.Close() db.acceptWG.Wait() + db.WaitForClose(250 * time.Millisecond) db.CloseAllConnections() tmpDir := path.Dir(db.socketFile) @@ -213,7 +214,7 @@ func (db *DB) Close() { // CloseAllConnections can be used to provoke MySQL client errors for open // connections. -// Make sure to call WaitForShutdown() as well. +// Make sure to call WaitForClose() as well. func (db *DB) CloseAllConnections() { db.mu.Lock() defer db.mu.Unlock() diff --git a/go/mysql/filepos_gtid.go b/go/mysql/filepos_gtid.go new file mode 100644 index 00000000000..9894e405494 --- /dev/null +++ b/go/mysql/filepos_gtid.go @@ -0,0 +1,133 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "fmt" + "strings" +) + +const filePosFlavorID = "FilePos" + +// parsefilePosGTID is registered as a GTID parser. +func parseFilePosGTID(s string) (GTID, error) { + // Split into parts. + parts := strings.Split(s, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid FilePos GTID (%v): expecting file:pos", s) + } + + return filePosGTID{ + file: parts[0], + pos: parts[1], + }, nil +} + +// parseFilePosGTIDSet is registered as a GTIDSet parser. +func parseFilePosGTIDSet(s string) (GTIDSet, error) { + gtid, err := parseFilePosGTID(s) + if err != nil { + return nil, err + } + return gtid.(filePosGTID), err +} + +// filePosGTID implements GTID. +type filePosGTID struct { + file, pos string +} + +// String implements GTID.String(). +func (gtid filePosGTID) String() string { + return gtid.file + ":" + gtid.pos +} + +// Flavor implements GTID.Flavor(). +func (gtid filePosGTID) Flavor() string { + return filePosFlavorID +} + +// SequenceDomain implements GTID.SequenceDomain(). +func (gtid filePosGTID) SequenceDomain() interface{} { + return nil +} + +// SourceServer implements GTID.SourceServer(). +func (gtid filePosGTID) SourceServer() interface{} { + return nil +} + +// SequenceNumber implements GTID.SequenceNumber(). +func (gtid filePosGTID) SequenceNumber() interface{} { + return nil +} + +// GTIDSet implements GTID.GTIDSet(). +func (gtid filePosGTID) GTIDSet() GTIDSet { + return gtid +} + +// ContainsGTID implements GTIDSet.ContainsGTID(). +func (gtid filePosGTID) ContainsGTID(other GTID) bool { + if other == nil { + return true + } + filePosOther, ok := other.(filePosGTID) + if !ok { + return false + } + if filePosOther.file < gtid.file { + return true + } + if filePosOther.file > gtid.file { + return false + } + return filePosOther.pos <= gtid.pos +} + +// Contains implements GTIDSet.Contains(). +func (gtid filePosGTID) Contains(other GTIDSet) bool { + if other == nil { + return true + } + filePosOther, _ := other.(filePosGTID) + return gtid.ContainsGTID(filePosOther) +} + +// Equal implements GTIDSet.Equal(). +func (gtid filePosGTID) Equal(other GTIDSet) bool { + filePosOther, ok := other.(filePosGTID) + if !ok { + return false + } + return gtid == filePosOther +} + +// AddGTID implements GTIDSet.AddGTID(). +func (gtid filePosGTID) AddGTID(other GTID) GTIDSet { + filePosOther, ok := other.(filePosGTID) + if !ok { + return gtid + } + return filePosOther +} + +func init() { + gtidParsers[filePosFlavorID] = parseFilePosGTID + gtidSetParsers[filePosFlavorID] = parseFilePosGTIDSet + flavors[filePosFlavorID] = newFilePosFlavor +} diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index eb337e102b5..4be34fd54da 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -71,7 +71,7 @@ type flavor interface { // resetReplicationCommands returns the commands to completely reset // replication on the host. - resetReplicationCommands() []string + resetReplicationCommands(c *Conn) []string // setSlavePositionCommands returns the commands to set the // replication position at which the slave will resume. @@ -99,7 +99,14 @@ type flavor interface { disableBinlogPlaybackCommand() string } -// fillFlavor fills in c.Flavor based on c.ServerVersion. +// flavors maps flavor names to their implementation. +// Flavors need to register only if they support being specified in the +// connection parameters. +var flavors = make(map[string]func() flavor) + +// fillFlavor fills in c.Flavor. If the params specify the flavor, +// that is used. Otherwise, we auto-detect. +// // This is the same logic as the ConnectorJ java client. We try to recognize // MariaDB as much as we can, but default to MySQL. // @@ -109,7 +116,12 @@ type flavor interface { // Note on such servers, 'select version()' would return 10.0.21-MariaDB-... // as well (not matching what c.ServerVersion is, but matching after we remove // the prefix). -func (c *Conn) fillFlavor() { +func (c *Conn) fillFlavor(params *ConnParams) { + if flavorFunc := flavors[params.Flavor]; flavorFunc != nil { + c.flavor = flavorFunc() + return + } + if strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix) { c.ServerVersion = c.ServerVersion[len(mariaDBReplicationHackPrefix):] c.flavor = mariadbFlavor{} @@ -179,7 +191,7 @@ func (c *Conn) ReadBinlogEvent() (BinlogEvent, error) { // ResetReplicationCommands returns the commands to completely reset // replication on the host. func (c *Conn) ResetReplicationCommands() []string { - return c.flavor.resetReplicationCommands() + return c.flavor.resetReplicationCommands(c) } // SetSlavePositionCommands returns the commands to set the diff --git a/go/mysql/flavor_filepos.go b/go/mysql/flavor_filepos.go new file mode 100644 index 00000000000..fe1f0e54a83 --- /dev/null +++ b/go/mysql/flavor_filepos.go @@ -0,0 +1,240 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "errors" + "fmt" + "io" + "strconv" + "time" + + "golang.org/x/net/context" +) + +type filePosFlavor struct { + format BinlogFormat + file string + savedEvent BinlogEvent +} + +// newFilePosFlavor creates a new filePos flavor. +func newFilePosFlavor() flavor { + return &filePosFlavor{} +} + +// masterGTIDSet is part of the Flavor interface. +func (flv *filePosFlavor) masterGTIDSet(c *Conn) (GTIDSet, error) { + qr, err := c.ExecuteFetch("SHOW SLAVE STATUS", 100, true /* wantfields */) + if err != nil { + return nil, err + } + if len(qr.Rows) == 0 { + qr, err = c.ExecuteFetch("SHOW MASTER STATUS", 100, true /* wantfields */) + if err != nil { + return nil, err + } + if len(qr.Rows) == 0 { + return nil, errors.New("no master or slave status") + } + resultMap, err := resultToMap(qr) + if err != nil { + return nil, err + } + return filePosGTID{ + file: resultMap["File"], + pos: resultMap["Position"], + }, nil + } + + resultMap, err := resultToMap(qr) + if err != nil { + return nil, err + } + return filePosGTID{ + file: resultMap["Relay_Master_Log_File"], + pos: resultMap["Exec_Master_Log_Pos"], + }, nil +} + +func (flv *filePosFlavor) startSlaveCommand() string { + return "unsupported" +} + +func (flv *filePosFlavor) stopSlaveCommand() string { + return "unsupported" +} + +// sendBinlogDumpCommand is part of the Flavor interface. +func (flv *filePosFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error { + rpos, ok := startPos.GTIDSet.(filePosGTID) + if !ok { + return fmt.Errorf("startPos.GTIDSet is wrong type - expected filePosGTID, got: %#v", startPos.GTIDSet) + } + + pos, err := strconv.Atoi(rpos.pos) + if err != nil { + return fmt.Errorf("invalid position: %v", startPos.GTIDSet) + } + flv.file = rpos.file + + return c.WriteComBinlogDump(slaveID, rpos.file, uint32(pos), 0) +} + +// readBinlogEvent is part of the Flavor interface. +func (flv *filePosFlavor) readBinlogEvent(c *Conn) (BinlogEvent, error) { + if ret := flv.savedEvent; ret != nil { + flv.savedEvent = nil + return ret, nil + } + + for { + result, err := c.ReadPacket() + if err != nil { + return nil, err + } + switch result[0] { + case EOFPacket: + return nil, NewSQLError(CRServerLost, SSUnknownSQLState, "%v", io.EOF) + case ErrPacket: + return nil, ParseErrorPacket(result) + } + + event := &filePosBinlogEvent{binlogEvent: binlogEvent(result[1:])} + switch event.Type() { + case eGTIDEvent, eAnonymousGTIDEvent, ePreviousGTIDsEvent, eMariaGTIDListEvent: + // Don't transmit fake or irrelevant events because we should not + // resume replication at these positions. + continue + case eMariaGTIDEvent: + // Copied from mariadb flavor. + const FLStandalone = 1 + flags2 := result[8+4] + // This means that it's also a BEGIN event. + if flags2&FLStandalone == 0 { + return newFilePosQueryEvent("begin", event.Timestamp()), nil + } + // Otherwise, don't send this event. + continue + case eFormatDescriptionEvent: + format, err := event.Format() + if err != nil { + return nil, err + } + flv.format = format + case eRotateEvent: + if !flv.format.IsZero() { + stripped, _, _ := event.StripChecksum(flv.format) + _, flv.file = stripped.(*filePosBinlogEvent).rotate(flv.format) + // No need to transmit. Just update the internal position for the next event. + continue + } + case eXIDEvent, eQueryEvent, eTableMapEvent, + eWriteRowsEventV0, eWriteRowsEventV1, eWriteRowsEventV2, + eDeleteRowsEventV0, eDeleteRowsEventV1, eDeleteRowsEventV2, + eUpdateRowsEventV0, eUpdateRowsEventV1, eUpdateRowsEventV2: + flv.savedEvent = event + return newFilePosGTIDEvent(flv.file, event.nextPosition(flv.format), event.Timestamp()), nil + default: + // For unrecognized events, send a fake "repair" event so that + // the position gets transmitted. + if !flv.format.IsZero() { + if v := event.nextPosition(flv.format); v != 0 { + flv.savedEvent = newFilePosQueryEvent("repair", event.Timestamp()) + return newFilePosGTIDEvent(flv.file, v, event.Timestamp()), nil + } + } + } + return event, nil + } +} + +// resetReplicationCommands is part of the Flavor interface. +func (flv *filePosFlavor) resetReplicationCommands(c *Conn) []string { + return []string{ + "unsupported", + } +} + +// setSlavePositionCommands is part of the Flavor interface. +func (flv *filePosFlavor) setSlavePositionCommands(pos Position) []string { + return []string{ + "unsupported", + } +} + +// setSlavePositionCommands is part of the Flavor interface. +func (flv *filePosFlavor) changeMasterArg() string { + return "unsupported" +} + +// status is part of the Flavor interface. +func (flv *filePosFlavor) status(c *Conn) (SlaveStatus, error) { + qr, err := c.ExecuteFetch("SHOW SLAVE STATUS", 100, true /* wantfields */) + if err != nil { + return SlaveStatus{}, err + } + if len(qr.Rows) == 0 { + // The query returned no data, meaning the server + // is not configured as a slave. + return SlaveStatus{}, ErrNotSlave + } + + resultMap, err := resultToMap(qr) + if err != nil { + return SlaveStatus{}, err + } + + status := parseSlaveStatus(resultMap) + status.Position.GTIDSet = filePosGTID{ + file: resultMap["Relay_Master_Log_File"], + pos: resultMap["Exec_Master_Log_Pos"], + } + return status, nil +} + +// waitUntilPositionCommand is part of the Flavor interface. +func (flv *filePosFlavor) waitUntilPositionCommand(ctx context.Context, pos Position) (string, error) { + filePosPos, ok := pos.GTIDSet.(filePosGTID) + if !ok { + return "", fmt.Errorf("Position is not filePos compatible: %#v", pos.GTIDSet) + } + + if deadline, ok := ctx.Deadline(); ok { + timeout := time.Until(deadline) + if timeout <= 0 { + return "", fmt.Errorf("timed out waiting for position %v", pos) + } + return fmt.Sprintf("SELECT MASTER_POS_WAIT('%s', %s, %.6f)", filePosPos.file, filePosPos.pos, timeout.Seconds()), nil + } + + return fmt.Sprintf("SELECT MASTER_POS_WAIT('%s', %s)", filePosPos.file, filePosPos.pos), nil +} + +func (*filePosFlavor) startSlaveUntilAfter(pos Position) string { + return "unsupported" +} + +// enableBinlogPlaybackCommand is part of the Flavor interface. +func (*filePosFlavor) enableBinlogPlaybackCommand() string { + return "" +} + +// disableBinlogPlaybackCommand is part of the Flavor interface. +func (*filePosFlavor) disableBinlogPlaybackCommand() string { + return "" +} diff --git a/go/mysql/flavor_mariadb.go b/go/mysql/flavor_mariadb.go index 98174aa9b48..397c0e63d2b 100644 --- a/go/mysql/flavor_mariadb.go +++ b/go/mysql/flavor_mariadb.go @@ -83,14 +83,17 @@ func (mariadbFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Pos } // resetReplicationCommands is part of the Flavor interface. -func (mariadbFlavor) resetReplicationCommands() []string { - return []string{ +func (mariadbFlavor) resetReplicationCommands(c *Conn) []string { + resetCommands := []string{ "STOP SLAVE", "RESET SLAVE ALL", // "ALL" makes it forget master host:port. "RESET MASTER", "SET GLOBAL gtid_slave_pos = ''", - "SET GLOBAL rpl_semi_sync_master_enabled = false, GLOBAL rpl_semi_sync_slave_enabled = false", // semi-sync will be enabled if needed when slave is started. } + if c.SemiSyncExtensionLoaded() { + resetCommands = append(resetCommands, "SET GLOBAL rpl_semi_sync_master_enabled = false, GLOBAL rpl_semi_sync_slave_enabled = false") // semi-sync will be enabled if needed when slave is started. + } + return resetCommands } // setSlavePositionCommands is part of the Flavor interface. diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 6ef3a34eb38..44127ed629c 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -66,13 +66,16 @@ func (mysqlFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Posit } // resetReplicationCommands is part of the Flavor interface. -func (mysqlFlavor) resetReplicationCommands() []string { - return []string{ +func (mysqlFlavor) resetReplicationCommands(c *Conn) []string { + resetCommands := []string{ "STOP SLAVE", "RESET SLAVE ALL", // "ALL" makes it forget master host:port. "RESET MASTER", // This will also clear gtid_executed and gtid_purged. - "SET GLOBAL rpl_semi_sync_master_enabled = false, GLOBAL rpl_semi_sync_slave_enabled = false", // semi-sync will be enabled if needed when slave is started. } + if c.SemiSyncExtensionLoaded() { + resetCommands = append(resetCommands, "SET GLOBAL rpl_semi_sync_master_enabled = false, GLOBAL rpl_semi_sync_slave_enabled = false") // semi-sync will be enabled if needed when slave is started. + } + return resetCommands } // setSlavePositionCommands is part of the Flavor interface. diff --git a/go/mysql/replication.go b/go/mysql/replication.go index e45b31d96e5..dcc4c5e20c2 100644 --- a/go/mysql/replication.go +++ b/go/mysql/replication.go @@ -67,3 +67,13 @@ func (c *Conn) WriteComBinlogDumpGTID(serverID uint32, binlogFilename string, bi } return nil } + +// SemiSyncExtensionLoaded checks if the semisync extension has been loaded. +// It should work for both MariaDB and MySQL. +func (c *Conn) SemiSyncExtensionLoaded() bool { + qr, err := c.ExecuteFetch("SHOW GLOBAL VARIABLES LIKE 'rpl_semi_sync%'", 10, false) + if err != nil { + return false + } + return len(qr.Rows) >= 1 +} diff --git a/go/mysql/server.go b/go/mysql/server.go index 213af75cb90..202aa732f55 100644 --- a/go/mysql/server.go +++ b/go/mysql/server.go @@ -37,7 +37,7 @@ import ( const ( // DefaultServerVersion is the default server version we're sending to the client. // Can be changed. - DefaultServerVersion = "5.5.10-Vitess" + DefaultServerVersion = "5.7.9-Vitess" // timing metric keys connectTimingKey = "Connect" diff --git a/go/netutil/netutil.go b/go/netutil/netutil.go index a97be114c81..937e2b46ced 100644 --- a/go/netutil/netutil.go +++ b/go/netutil/netutil.go @@ -154,7 +154,7 @@ func FullyQualifiedHostname() (string, error) { // 127.0.0.1 localhost.localdomain localhost // If the FQDN isn't returned by this function, check the order in the entry // in your /etc/hosts file. - return strings.TrimRight(resolvedHostnames[0], "."), nil + return strings.TrimSuffix(resolvedHostnames[0], "."), nil } // FullyQualifiedHostnameOrPanic is the same as FullyQualifiedHostname diff --git a/go/pools/resource_pool.go b/go/pools/resource_pool.go index 81055e4d0cb..63b8309137b 100644 --- a/go/pools/resource_pool.go +++ b/go/pools/resource_pool.go @@ -38,6 +38,9 @@ var ( // ErrTimeout is returned if a resource get times out. ErrTimeout = errors.New("resource pool timed out") + // ErrCtxTimeout is returned if a ctx is already expired by the time the resource pool is used + ErrCtxTimeout = errors.New("resource pool context already expired") + prefillTimeout = 30 * time.Second ) @@ -198,7 +201,7 @@ func (rp *ResourcePool) get(ctx context.Context) (resource Resource, err error) // If ctx has already expired, avoid racing with rp's resource channel. select { case <-ctx.Done(): - return nil, ErrTimeout + return nil, ErrCtxTimeout default: } diff --git a/go/pools/resource_pool_test.go b/go/pools/resource_pool_test.go index 2ec6f68c8b6..f3950e5e23e 100644 --- a/go/pools/resource_pool_test.go +++ b/go/pools/resource_pool_test.go @@ -639,7 +639,7 @@ func TestExpired(t *testing.T) { p.Put(r) } cancel() - want := "resource pool timed out" + want := "resource pool context already expired" if err == nil || err.Error() != want { t.Errorf("got %v, want %s", err, want) } diff --git a/go/stats/prometheusbackend/prometheusbackend.go b/go/stats/prometheusbackend/prometheusbackend.go index fada0aaa4a2..f975a12f08b 100644 --- a/go/stats/prometheusbackend/prometheusbackend.go +++ b/go/stats/prometheusbackend/prometheusbackend.go @@ -82,7 +82,7 @@ func (be PromBackend) publishPrometheusMetric(name string, v expvar.Var) { newMultiTimingsCollector(st, be.buildPromName(name)) case *stats.Histogram: newHistogramCollector(st, be.buildPromName(name)) - case *stats.String, stats.StringFunc, stats.StringMapFunc, *stats.Rates: + case *stats.String, stats.StringFunc, stats.StringMapFunc, *stats.Rates, *stats.RatesFunc: // Silently ignore these types since they don't make sense to // export to Prometheus' data model. default: diff --git a/go/stats/rates.go b/go/stats/rates.go index 2c7a556d3f0..7aa4f7d3ce7 100644 --- a/go/stats/rates.go +++ b/go/stats/rates.go @@ -182,3 +182,32 @@ func (rt *Rates) String() string { } return string(data) } + +type RatesFunc struct { + F func() map[string][]float64 + help string +} + +func NewRateFunc(name string, help string, f func() map[string][]float64) *RatesFunc { + c := &RatesFunc{ + F: f, + help: help, + } + + if name != "" { + publish(name, c) + } + return c +} + +func (rf *RatesFunc) Help() string { + return rf.help +} + +func (rf *RatesFunc) String() string { + data, err := json.Marshal(rf.F()) + if err != nil { + data, _ = json.Marshal(err.Error()) + } + return string(data) +} diff --git a/go/test/endtoend/README.md b/go/test/endtoend/README.md new file mode 100644 index 00000000000..a8cdcca990d --- /dev/null +++ b/go/test/endtoend/README.md @@ -0,0 +1,50 @@ +This document describe the testing strategy we use for all Vitess components, and the progression in scope / complexity. + +As Vitess developers, our goal is to have great end to end test coverage. In the past, these tests were mostly written in python 2.7 is coming to end of life we are moving all of those into GO. + + +## End to End Tests + +These tests are meant to test end-to-end behaviors of the Vitess ecosystem, and complement the unit tests. For instance, we test each RPC interaction independently (client to vtgate, vtgate to vttablet, vttablet to MySQL, see previous sections). But is also good to have an end-to-end test that validates everything works together. + +These tests almost always launch a topology service, a few mysqld instances, a few vttablets, a vtctld process, a few vtgates, ... They use the real production processes, and real replication. This setup is mandatory for properly testing re-sharding, cluster operations, ... They all however run on the same machine, so they might be limited by the environment. + + +## Strategy + +All the end to end test are placed under path go/test/endtoend. +The main purpose of grouping them together is to make sure we have single place for reference and to combine similar test to run them in the same cluster and save test running time. + +### Setup +All the tests should be launching a real cluster just like the production setup and execute the tests on that setup followed by a teardown of all the services. + +The cluster launch functions are provided under go/test/endtoend/cluster. This is still work in progress so feel free to add new function as required or update the existing ones. + +In general the cluster is build in following order +- Define Keyspace +- Define Shards +- Start topology service [default etcd] +- Start vtctld client +- Start required mysqld instances +- Start corresponding vttablets (atleast 1 master and 1 replica) +- Start Vtgate + +A good example to refer will be go/test/endtoend/clustertest + +## Progress +So far we have converted the following Python end to end test cases +- Keyspace tests +- mysqlctl tests +- sharded tests +- tabletmanager tests +- vtgate v3 tests + +### In-progress +- Inital sharding +- resharding +- vsplit + + +After a Python test is migrated in Go it will be removed from end to end ci test run by updating the shard value to 5 in `test/config.json` + + diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go new file mode 100644 index 00000000000..5315adbd9a4 --- /dev/null +++ b/go/test/endtoend/cluster/cluster_process.go @@ -0,0 +1,454 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "flag" + "fmt" + "math/rand" + "os" + "os/exec" + "path" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// DefaultCell : If no cell name is passed, then use following +const DefaultCell = "zone1" + +var ( + keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") +) + +// LocalProcessCluster Testcases need to use this to iniate a cluster +type LocalProcessCluster struct { + Keyspaces []Keyspace + Cell string + BaseTabletUID int + Hostname string + TopoPort int + TmpDirectory string + OriginalVTDATAROOT string + CurrentVTDATAROOT string + + VtgateMySQLPort int + VtgateGrpcPort int + VtctldHTTPPort int + + // standalone executable + VtctlclientProcess VtctlClientProcess + VtctlProcess VtctlProcess + + // background executable processes + TopoProcess EtcdProcess + VtctldProcess VtctldProcess + VtgateProcess VtgateProcess + + nextPortForProcess int + + //Extra arguments for vtTablet + VtTabletExtraArgs []string + + //Extra arguments for vtGate + VtGateExtraArgs []string + + VtctldExtraArgs []string + + EnableSemiSync bool +} + +// Keyspace : Cluster accepts keyspace to launch it +type Keyspace struct { + Name string + SchemaSQL string + VSchema string + Shards []Shard +} + +// Shard with associated vttablets +type Shard struct { + Name string + Vttablets []Vttablet +} + +// Vttablet stores the properties needed to start a vttablet process +type Vttablet struct { + Type string + TabletUID int + HTTPPort int + GrpcPort int + MySQLPort int + Alias string + + // background executable processes + MysqlctlProcess MysqlctlProcess + VttabletProcess *VttabletProcess +} + +// StartTopo starts topology server +func (cluster *LocalProcessCluster) StartTopo() (err error) { + if cluster.Cell == "" { + cluster.Cell = DefaultCell + } + cluster.TopoPort = cluster.GetAndReservePort() + cluster.TmpDirectory = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/tmp_%d", cluster.GetAndReservePort())) + cluster.TopoProcess = *EtcdProcessInstance(cluster.TopoPort, cluster.GetAndReservePort(), cluster.Hostname, "global") + log.Info(fmt.Sprintf("Starting etcd server on port : %d", cluster.TopoPort)) + if err = cluster.TopoProcess.Setup(); err != nil { + log.Error(err.Error()) + return + } + + log.Info("Creating topo dirs") + if err = cluster.TopoProcess.ManageTopoDir("mkdir", "/vitess/global"); err != nil { + log.Error(err.Error()) + return + } + + if err = cluster.TopoProcess.ManageTopoDir("mkdir", "/vitess/"+cluster.Cell); err != nil { + log.Error(err.Error()) + return + } + + log.Info("Adding cell info") + cluster.VtctlProcess = *VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname) + if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil { + log.Error(err) + return + } + + cluster.VtctldProcess = *VtctldProcessInstance(cluster.GetAndReservePort(), cluster.GetAndReservePort(), cluster.TopoProcess.Port, cluster.Hostname, cluster.TmpDirectory) + log.Info(fmt.Sprintf("Starting vtctld server on port : %d", cluster.VtctldProcess.Port)) + cluster.VtctldHTTPPort = cluster.VtctldProcess.Port + if err = cluster.VtctldProcess.Setup(cluster.Cell, cluster.VtctldExtraArgs...); err != nil { + + log.Error(err.Error()) + return + } + + cluster.VtctlclientProcess = *VtctlClientProcessInstance("localhost", cluster.VtctldProcess.GrpcPort, cluster.TmpDirectory) + return +} + +// StartUnshardedKeyspace starts unshared keyspace with shard name as "0" +func (cluster *LocalProcessCluster) StartUnshardedKeyspace(keyspace Keyspace, replicaCount int, rdonly bool) error { + return cluster.StartKeyspace(keyspace, []string{"0"}, replicaCount, rdonly) +} + +// StartKeyspace starts required number of shard and the corresponding tablets +// keyspace : struct containing keyspace name, Sqlschema to apply, VSchema to apply +// shardName : list of shard names +// replicaCount: total number of replicas excluding master and rdonly +// rdonly: whether readonly tablets needed +func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames []string, replicaCount int, rdonly bool) (err error) { + totalTabletsRequired := replicaCount + 1 // + 1 is for master + if rdonly { + totalTabletsRequired = totalTabletsRequired + 1 // + 1 for rdonly + } + + log.Info("Starting keyspace : " + keyspace.Name) + _ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) + var mysqlctlProcessList []*exec.Cmd + for _, shardName := range shardNames { + shard := &Shard{ + Name: shardName, + } + log.Info("Starting shard : " + shardName) + mysqlctlProcessList = []*exec.Cmd{} + for i := 0; i < totalTabletsRequired; i++ { + // instantiate vttablet object with reserved ports + tabletUID := cluster.GetAndReserveTabletUID() + tablet := &Vttablet{ + TabletUID: tabletUID, + HTTPPort: cluster.GetAndReservePort(), + GrpcPort: cluster.GetAndReservePort(), + MySQLPort: cluster.GetAndReservePort(), + Alias: fmt.Sprintf("%s-%010d", cluster.Cell, tabletUID), + } + if i == 0 { // Make the first one as master + tablet.Type = "master" + } else if i == totalTabletsRequired-1 && rdonly { // Make the last one as rdonly if rdonly flag is passed + tablet.Type = "rdonly" + } + // Start Mysqlctl process + log.Info(fmt.Sprintf("Starting mysqlctl for table uid %d, mysql port %d", tablet.TabletUID, tablet.MySQLPort)) + tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory) + if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + log.Error(err.Error()) + return err + } else { + mysqlctlProcessList = append(mysqlctlProcessList, proc) + } + + // start vttablet process + tablet.VttabletProcess = VttabletProcessInstance(tablet.HTTPPort, + tablet.GrpcPort, + tablet.TabletUID, + cluster.Cell, + shardName, + keyspace.Name, + cluster.VtctldProcess.Port, + tablet.Type, + cluster.TopoProcess.Port, + cluster.Hostname, + cluster.TmpDirectory, + cluster.VtTabletExtraArgs, + cluster.EnableSemiSync) + tablet.Alias = tablet.VttabletProcess.TabletPath + + shard.Vttablets = append(shard.Vttablets, *tablet) + } + + // wait till all mysqlctl is instantiated + for _, proc := range mysqlctlProcessList { + if err = proc.Wait(); err != nil { + log.Errorf("Unable to start mysql , error %v", err.Error()) + return err + } + } + for _, tablet := range shard.Vttablets { + if _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil { + log.Error(err.Error()) + return + } + + log.Info(fmt.Sprintf("Starting vttablet for tablet uid %d, grpc port %d", tablet.TabletUID, tablet.GrpcPort)) + + if err = tablet.VttabletProcess.Setup(); err != nil { + log.Error(err.Error()) + return + } + } + + // Make first tablet as master + if err = cluster.VtctlclientProcess.InitShardMaster(keyspace.Name, shardName, cluster.Cell, shard.Vttablets[0].TabletUID); err != nil { + log.Error(err.Error()) + return + } + keyspace.Shards = append(keyspace.Shards, *shard) + } + // if the keyspace is present then append the shard info + existingKeyspace := false + for idx, ks := range cluster.Keyspaces { + if ks.Name == keyspace.Name { + cluster.Keyspaces[idx].Shards = append(cluster.Keyspaces[idx].Shards, keyspace.Shards...) + existingKeyspace = true + } + } + if !existingKeyspace { + cluster.Keyspaces = append(cluster.Keyspaces, keyspace) + } + + // Apply Schema SQL + if keyspace.SchemaSQL != "" { + if err = cluster.VtctlclientProcess.ApplySchema(keyspace.Name, keyspace.SchemaSQL); err != nil { + log.Error(err.Error()) + return + } + } + + //Apply VSchema + if keyspace.VSchema != "" { + if err = cluster.VtctlclientProcess.ApplyVSchema(keyspace.Name, keyspace.VSchema); err != nil { + log.Error(err.Error()) + return + } + } + + log.Info("Done creating keyspace : " + keyspace.Name) + return +} + +// StartVtgate starts vtgate +func (cluster *LocalProcessCluster) StartVtgate() (err error) { + vtgateHTTPPort := cluster.GetAndReservePort() + vtgateGrpcPort := cluster.GetAndReservePort() + cluster.VtgateMySQLPort = cluster.GetAndReservePort() + log.Info(fmt.Sprintf("Starting vtgate on port %d", vtgateHTTPPort)) + cluster.VtgateProcess = *VtgateProcessInstance( + vtgateHTTPPort, + vtgateGrpcPort, + cluster.VtgateMySQLPort, + cluster.Cell, + cluster.Cell, + cluster.Hostname, + "MASTER,REPLICA", + cluster.TopoProcess.Port, + cluster.TmpDirectory, + cluster.VtGateExtraArgs) + + log.Info(fmt.Sprintf("Vtgate started, connect to mysql using : mysql -h 127.0.0.1 -P %d", cluster.VtgateMySQLPort)) + if err = cluster.VtgateProcess.Setup(); err != nil { + return err + } + if err = cluster.WaitForTabletsToHealthyInVtgate(); err != nil { + return err + } + return nil +} + +// NewCluster instantiates a new cluster +func NewCluster(cell string, hostname string) *LocalProcessCluster { + cluster := &LocalProcessCluster{Cell: cell, Hostname: hostname} + cluster.OriginalVTDATAROOT = os.Getenv("VTDATAROOT") + cluster.CurrentVTDATAROOT = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("vtroot_%d", cluster.GetAndReservePort())) + _ = createDirectory(cluster.CurrentVTDATAROOT, 0700) + _ = os.Setenv("VTDATAROOT", cluster.CurrentVTDATAROOT) + rand.Seed(time.Now().UTC().UnixNano()) + return cluster +} + +// ReStartVtgate starts vtgate with updated configs +func (cluster *LocalProcessCluster) ReStartVtgate() (err error) { + err = cluster.VtgateProcess.TearDown() + if err != nil { + log.Error(err.Error()) + return + } + err = cluster.StartVtgate() + if err != nil { + log.Error(err.Error()) + return + } + return err +} + +// WaitForTabletsToHealthyInVtgate waits for all tablets in all shards to be healthy as per vtgate +func (cluster *LocalProcessCluster) WaitForTabletsToHealthyInVtgate() (err error) { + var isRdOnlyPresent bool + for _, keyspace := range cluster.Keyspaces { + for _, shard := range keyspace.Shards { + isRdOnlyPresent = false + if err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", keyspace.Name, shard.Name)); err != nil { + return err + } + if err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.replica", keyspace.Name, shard.Name)); err != nil { + return err + } + for _, tablet := range shard.Vttablets { + if tablet.Type == "rdonly" { + isRdOnlyPresent = true + } + } + if isRdOnlyPresent { + err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspace.Name, shard.Name)) + } + if err != nil { + return err + } + } + } + return nil +} + +// Teardown brings down the cluster by invoking teardown for individual processes +func (cluster *LocalProcessCluster) Teardown() { + if err := cluster.VtgateProcess.TearDown(); err != nil { + log.Errorf("Error in vtgate teardown - %s", err.Error()) + } + mysqlctlProcessList := []*exec.Cmd{} + for _, keyspace := range cluster.Keyspaces { + for _, shard := range keyspace.Shards { + for _, tablet := range shard.Vttablets { + if proc, err := tablet.MysqlctlProcess.StopProcess(); err != nil { + log.Errorf("Error in mysqlctl teardown - %s", err.Error()) + } else { + mysqlctlProcessList = append(mysqlctlProcessList, proc) + } + + if err := tablet.VttabletProcess.TearDown(); err != nil { + log.Errorf("Error in vttablet teardown - %s", err.Error()) + } + } + } + } + + for _, proc := range mysqlctlProcessList { + if err := proc.Wait(); err != nil { + log.Errorf("Error in mysqlctl teardown wait - %s", err.Error()) + } + } + + if err := cluster.VtctldProcess.TearDown(); err != nil { + log.Errorf("Error in vtctld teardown - %s", err.Error()) + } + + if err := cluster.TopoProcess.TearDown(cluster.Cell, cluster.OriginalVTDATAROOT, cluster.CurrentVTDATAROOT, *keepData); err != nil { + log.Errorf("Error in etcd teardown - %s", err.Error()) + } +} + +// GetAndReservePort gives port for required process +func (cluster *LocalProcessCluster) GetAndReservePort() int { + if cluster.nextPortForProcess == 0 { + cluster.nextPortForProcess = getRandomNumber(20000, 15000) + } + cluster.nextPortForProcess = cluster.nextPortForProcess + 1 + return cluster.nextPortForProcess +} + +// GetAndReserveTabletUID gives tablet uid +func (cluster *LocalProcessCluster) GetAndReserveTabletUID() int { + if cluster.BaseTabletUID == 0 { + cluster.BaseTabletUID = getRandomNumber(10000, 0) + } + cluster.BaseTabletUID = cluster.BaseTabletUID + 1 + return cluster.BaseTabletUID +} + +func getRandomNumber(maxNumber int32, baseNumber int) int { + return int(rand.Int31n(maxNumber)) + baseNumber +} + +// GetVttabletInstance create a new vttablet object +func (cluster *LocalProcessCluster) GetVttabletInstance(UID int) *Vttablet { + if UID == 0 { + UID = cluster.GetAndReserveTabletUID() + } + return &Vttablet{ + TabletUID: UID, + HTTPPort: cluster.GetAndReservePort(), + GrpcPort: cluster.GetAndReservePort(), + MySQLPort: cluster.GetAndReservePort(), + Type: "replica", + Alias: fmt.Sprintf("%s-%010d", cluster.Cell, UID), + } +} + +// StartVttablet start a new tablet +func (cluster *LocalProcessCluster) StartVttablet(tablet *Vttablet, servingStatus string, + supportBackup bool, cell string, keyspaceName string, hostname string, shardName string) error { + tablet.VttabletProcess = VttabletProcessInstance( + tablet.HTTPPort, + tablet.GrpcPort, + tablet.TabletUID, + cell, + shardName, + keyspaceName, + cluster.VtctldProcess.Port, + tablet.Type, + cluster.TopoProcess.Port, + hostname, + cluster.TmpDirectory, + cluster.VtTabletExtraArgs, + cluster.EnableSemiSync) + + tablet.VttabletProcess.SupportBackup = supportBackup + tablet.VttabletProcess.ServingStatus = servingStatus + return tablet.VttabletProcess.Setup() +} \ No newline at end of file diff --git a/go/test/endtoend/cluster/etcd_process.go b/go/test/endtoend/cluster/etcd_process.go new file mode 100644 index 00000000000..73d9f125774 --- /dev/null +++ b/go/test/endtoend/cluster/etcd_process.go @@ -0,0 +1,178 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "net/http" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// EtcdProcess is a generic handle for a running Etcd . +// It can be spawned manually +type EtcdProcess struct { + Name string + Binary string + DataDirectory string + ListenClientURL string + AdvertiseClientURL string + Port int + PeerPort int + Host string + VerifyURL string + PeerURL string + + proc *exec.Cmd + exit chan error +} + +// Setup spawns a new etcd service and initializes it with the defaults. +// The service is kept running in the background until TearDown() is called. +func (etcd *EtcdProcess) Setup() (err error) { + etcd.proc = exec.Command( + etcd.Binary, + "--name", etcd.Name, + "--data-dir", etcd.DataDirectory, + "--listen-client-urls", etcd.ListenClientURL, + "--advertise-client-urls", etcd.AdvertiseClientURL, + "--initial-advertise-peer-urls", etcd.PeerURL, + "--listen-peer-urls", etcd.PeerURL, + "--initial-cluster", fmt.Sprintf("%s=%s", etcd.Name, etcd.PeerURL), + ) + + errFile, _ := os.Create(path.Join(etcd.DataDirectory, "etcd-stderr.txt")) + etcd.proc.Stderr = errFile + + etcd.proc.Env = append(etcd.proc.Env, os.Environ()...) + + log.Infof("%v %v", strings.Join(etcd.proc.Args, " ")) + println("Starting etcd with args " + strings.Join(etcd.proc.Args, " ")) + err = etcd.proc.Start() + if err != nil { + return + } + + etcd.exit = make(chan error) + go func() { + etcd.exit <- etcd.proc.Wait() + }() + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + if etcd.IsHealthy() { + return + } + select { + case err := <-etcd.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", etcd.Binary, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", etcd.Binary, <-etcd.exit) +} + +// TearDown shutdowns the running mysqld service +func (etcd *EtcdProcess) TearDown(Cell string, originalVtRoot string, currentRoot string, keepdata bool) error { + if etcd.proc == nil || etcd.exit == nil { + return nil + } + + etcd.removeTopoDirectories(Cell) + + // Attempt graceful shutdown with SIGTERM first + _ = etcd.proc.Process.Signal(syscall.SIGTERM) + if !*keepData { + _ = os.RemoveAll(etcd.DataDirectory) + _ = os.RemoveAll(currentRoot) + } + _ = os.Setenv("VTDATAROOT", originalVtRoot) + select { + case <-etcd.exit: + etcd.proc = nil + return nil + + case <-time.After(10 * time.Second): + etcd.proc.Process.Kill() + etcd.proc = nil + return <-etcd.exit + } + +} + +// IsHealthy function checks if etcd server is up and running +func (etcd *EtcdProcess) IsHealthy() bool { + resp, err := http.Get(etcd.VerifyURL) + if err != nil { + return false + } + if resp.StatusCode == 200 { + return true + } + return false +} + +func (etcd *EtcdProcess) removeTopoDirectories(Cell string) { + _ = etcd.ManageTopoDir("rmdir", "/vitess/global") + _ = etcd.ManageTopoDir("rmdir", "/vitess/"+Cell) +} + +// ManageTopoDir creates global and zone in etcd2 +func (etcd *EtcdProcess) ManageTopoDir(command string, directory string) (err error) { + url := etcd.VerifyURL + directory + payload := strings.NewReader(`{"dir":"true"}`) + if command == "mkdir" { + req, _ := http.NewRequest("PUT", url, payload) + req.Header.Add("content-type", "application/json") + _, err = http.DefaultClient.Do(req) + return err + } else if command == "rmdir" { + req, _ := http.NewRequest("DELETE", url+"?dir=true", payload) + _, err = http.DefaultClient.Do(req) + return err + } else { + return nil + } +} + +// EtcdProcessInstance returns a EtcdProcess handle for a etcd sevice, +// configured with the given Config. +// The process must be manually started by calling setup() +func EtcdProcessInstance(port int, peerPort int, hostname string, name string) *EtcdProcess { + etcd := &EtcdProcess{ + Name: name, + Binary: "etcd", + Port: port, + Host: hostname, + PeerPort: peerPort, + } + + etcd.AdvertiseClientURL = fmt.Sprintf("http://%s:%d", etcd.Host, etcd.Port) + etcd.ListenClientURL = fmt.Sprintf("http://%s:%d", etcd.Host, etcd.Port) + etcd.DataDirectory = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("%s_%d", "etcd", port)) + etcd.VerifyURL = fmt.Sprintf("http://%s:%d/v2/keys", etcd.Host, etcd.Port) + etcd.PeerURL = fmt.Sprintf("http://%s:%d", hostname, peerPort) + return etcd +} diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go new file mode 100644 index 00000000000..14a5a7fe0f8 --- /dev/null +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -0,0 +1,155 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/log" +) + +// MysqlctlProcess is a generic handle for a running mysqlctl command . +// It can be spawned manually +type MysqlctlProcess struct { + Name string + Binary string + LogDirectory string + TabletUID int + MySQLPort int + InitDBFile string +} + +// InitDb executes mysqlctl command to add cell info +func (mysqlctl *MysqlctlProcess) InitDb() (err error) { + tmpProcess := exec.Command( + mysqlctl.Binary, + "-log_dir", mysqlctl.LogDirectory, + "-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID), + "-mysql_port", fmt.Sprintf("%d", mysqlctl.MySQLPort), + "init", + "-init_db_sql_file", mysqlctl.InitDBFile, + ) + return tmpProcess.Run() +} + +// Start executes mysqlctl command to start mysql instance +func (mysqlctl *MysqlctlProcess) Start() (err error) { + if tmpProcess, err := mysqlctl.StartProcess(); err != nil { + return err + } else { + return tmpProcess.Wait() + } +} + +// StartProcess starts the mysqlctl and returns the process reference +func (mysqlctl *MysqlctlProcess) StartProcess() (*exec.Cmd, error) { + tmpProcess := exec.Command( + mysqlctl.Binary, + "-log_dir", mysqlctl.LogDirectory, + "-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID), + "-mysql_port", fmt.Sprintf("%d", mysqlctl.MySQLPort), + "init", + "-init_db_sql_file", mysqlctl.InitDBFile, + ) + return tmpProcess, tmpProcess.Start() +} + +// Stop executes mysqlctl command to stop mysql instance +func (mysqlctl *MysqlctlProcess) Stop() (err error) { + if tmpProcess, err := mysqlctl.StopProcess(); err != nil { + return err + } else { + return tmpProcess.Wait() + } +} + +// StopProcess executes mysqlctl command to stop mysql instance and returns process reference +func (mysqlctl *MysqlctlProcess) StopProcess() (*exec.Cmd, error) { + tmpProcess := exec.Command( + mysqlctl.Binary, + "-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID), + "shutdown", + ) + return tmpProcess, tmpProcess.Start() +} + +// CleanupFiles clean the mysql files to make sure we can start the same process again +func (mysqlctl *MysqlctlProcess) CleanupFiles(tabletUID int) { + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/data", tabletUID))) + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/relay-logs", tabletUID))) + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/tmp", tabletUID))) + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/bin-logs", tabletUID))) + os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/innodb", tabletUID))) +} + +// MysqlCtlProcessInstance returns a Mysqlctl handle for mysqlctl process +// configured with the given Config. +func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctlProcess { + mysqlctl := &MysqlctlProcess{ + Name: "mysqlctl", + Binary: "mysqlctl", + LogDirectory: tmpDirectory, + InitDBFile: path.Join(os.Getenv("VTROOT"), "/config/init_db.sql"), + } + mysqlctl.MySQLPort = mySQLPort + mysqlctl.TabletUID = tabletUID + return mysqlctl +} + +// StartMySQL process +func StartMySQL(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error { + tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + err := tablet.MysqlctlProcess.Start() + if err != nil { + return err + } + return nil +} + +// StartMySQLAndGetConnection create a connection to tablet mysql +func StartMySQLAndGetConnection(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) (*mysql.Conn, error) { + tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory) + err := tablet.MysqlctlProcess.Start() + if err != nil { + return nil, err + } + params := mysql.ConnParams{ + Uname: username, + UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tablet.TabletUID), "/mysql.sock"), + } + + conn, err := mysql.Connect(ctx, ¶ms) + return conn, err +} + +// ExecuteCommandWithOutput executes any mysqlctl command and returns output +func (mysqlctl *MysqlctlProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) { + tmpProcess := exec.Command( + mysqlctl.Binary, + args..., + ) + println(fmt.Sprintf("Executing mysqlctl with arguments %v", strings.Join(tmpProcess.Args, " "))) + log.Info(fmt.Sprintf("Executing mysqlctl with arguments %v", strings.Join(tmpProcess.Args, " "))) + resultByte, err := tmpProcess.CombinedOutput() + return string(resultByte), err +} diff --git a/go/test/endtoend/cluster/vtctl_process.go b/go/test/endtoend/cluster/vtctl_process.go new file mode 100644 index 00000000000..74511b5c70f --- /dev/null +++ b/go/test/endtoend/cluster/vtctl_process.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "os/exec" + "strings" + + "vitess.io/vitess/go/vt/log" +) + +// VtctlProcess is a generic handle for a running vtctl command . +// It can be spawned manually +type VtctlProcess struct { + Name string + Binary string + TopoImplementation string + TopoGlobalAddress string + TopoGlobalRoot string + TopoServerAddress string +} + +// AddCellInfo executes vtctl command to add cell info +func (vtctl *VtctlProcess) AddCellInfo(Cell string) (err error) { + tmpProcess := exec.Command( + vtctl.Binary, + "-topo_implementation", vtctl.TopoImplementation, + "-topo_global_server_address", vtctl.TopoGlobalAddress, + "-topo_global_root", vtctl.TopoGlobalRoot, + "AddCellInfo", + "-root", "/vitess/"+Cell, + "-server_address", vtctl.TopoServerAddress, + Cell, + ) + return tmpProcess.Run() +} + +// CreateKeyspace executes vtctl command to create keyspace +func (vtctl *VtctlProcess) CreateKeyspace(keyspace string) (err error) { + tmpProcess := exec.Command( + vtctl.Binary, + "-topo_implementation", vtctl.TopoImplementation, + "-topo_global_server_address", vtctl.TopoGlobalAddress, + "-topo_global_root", vtctl.TopoGlobalRoot, + "CreateKeyspace", keyspace, + ) + log.Info(fmt.Sprintf("Starting CreateKeyspace with arguments %v", strings.Join(tmpProcess.Args, " "))) + return tmpProcess.Run() +} + +// VtctlProcessInstance returns a VtctlProcess handle for vtctl process +// configured with the given Config. +// The process must be manually started by calling setup() +func VtctlProcessInstance(topoPort int, hostname string) *VtctlProcess { + vtctl := &VtctlProcess{ + Name: "vtctl", + Binary: "vtctl", + TopoImplementation: "etcd2", + TopoGlobalAddress: fmt.Sprintf("%s:%d", hostname, topoPort), + TopoGlobalRoot: "/vitess/global", + TopoServerAddress: fmt.Sprintf("%s:%d", hostname, topoPort), + } + return vtctl +} diff --git a/go/test/endtoend/cluster/vtctlclient_process.go b/go/test/endtoend/cluster/vtctlclient_process.go new file mode 100644 index 00000000000..f7287a80f3f --- /dev/null +++ b/go/test/endtoend/cluster/vtctlclient_process.go @@ -0,0 +1,107 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "os/exec" + "strings" + + "vitess.io/vitess/go/vt/log" +) + +// VtctlClientProcess is a generic handle for a running vtctlclient command . +// It can be spawned manually +type VtctlClientProcess struct { + Name string + Binary string + Server string + TempDirectory string + ZoneName string +} + +// InitShardMaster executes vtctlclient command to make one of tablet as master +func (vtctlclient *VtctlClientProcess) InitShardMaster(Keyspace string, Shard string, Cell string, TabletUID int) (err error) { + return vtctlclient.ExecuteCommand( + "InitShardMaster", + "-force", + fmt.Sprintf("%s/%s", Keyspace, Shard), + fmt.Sprintf("%s-%d", Cell, TabletUID)) +} + +// ApplySchema applies SQL schema to the keyspace +func (vtctlclient *VtctlClientProcess) ApplySchema(Keyspace string, SQL string) (err error) { + return vtctlclient.ExecuteCommand( + "ApplySchema", + "-sql", SQL, + Keyspace) +} + +// ApplyVSchema applies vitess schema (JSON format) to the keyspace +func (vtctlclient *VtctlClientProcess) ApplyVSchema(Keyspace string, JSON string) (err error) { + return vtctlclient.ExecuteCommand( + "ApplyVSchema", + "-vschema", JSON, + Keyspace, + ) +} + +// ExecuteCommand executes any vtctlclient command +func (vtctlclient *VtctlClientProcess) ExecuteCommand(args ...string) (err error) { + args = append([]string{"-server", vtctlclient.Server}, args...) + tmpProcess := exec.Command( + vtctlclient.Binary, + args..., + ) + println(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " "))) + log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " "))) + return tmpProcess.Run() +} + +// ExecuteCommandWithOutput executes any vtctlclient command and returns output +func (vtctlclient *VtctlClientProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) { + args = append([]string{"-server", vtctlclient.Server}, args...) + tmpProcess := exec.Command( + vtctlclient.Binary, + args..., + ) + println(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " "))) + log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " "))) + resultByte, err := tmpProcess.CombinedOutput() + return string(resultByte), err +} + +// VtctlClientProcessInstance returns a VtctlProcess handle for vtctlclient process +// configured with the given Config. +func VtctlClientProcessInstance(hostname string, grpcPort int, tmpDirectory string) *VtctlClientProcess { + vtctlclient := &VtctlClientProcess{ + Name: "vtctlclient", + Binary: "vtctlclient", + Server: fmt.Sprintf("%s:%d", hostname, grpcPort), + TempDirectory: tmpDirectory, + } + return vtctlclient +} + +// InitTablet initializes a tablet +func (vtctlclient *VtctlClientProcess) InitTablet(tablet *Vttablet, cell string, keyspaceName string, hostname string, shardName string) error { + return vtctlclient.ExecuteCommand( + "InitTablet", "-hostname", hostname, + "-port", fmt.Sprintf("%d", tablet.HTTPPort), "-allow_update", + "-keyspace", keyspaceName, "-shard", shardName, + fmt.Sprintf("%s-%010d", cell, tablet.TabletUID), "replica") +} diff --git a/go/test/endtoend/cluster/vtctld_process.go b/go/test/endtoend/cluster/vtctld_process.go new file mode 100644 index 00000000000..52a1ec7c45b --- /dev/null +++ b/go/test/endtoend/cluster/vtctld_process.go @@ -0,0 +1,174 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "net/http" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// VtctldProcess is a generic handle for a running vtctld . +// It can be spawned manually +type VtctldProcess struct { + Name string + Binary string + CommonArg VtctlProcess + WebDir string + WebDir2 string + ServiceMap string + BackupStorageImplementation string + FileBackupStorageRoot string + LogDir string + Port int + GrpcPort int + PidFile string + VerifyURL string + Directory string + + proc *exec.Cmd + exit chan error +} + +// Setup starts vtctld process with required arguements +func (vtctld *VtctldProcess) Setup(cell string, extraArgs ...string) (err error) { + _ = createDirectory(vtctld.LogDir, 0700) + _ = createDirectory(path.Join(vtctld.Directory, "backups"), 0700) + vtctld.proc = exec.Command( + vtctld.Binary, + "-enable_queries", + "-topo_implementation", vtctld.CommonArg.TopoImplementation, + "-topo_global_server_address", vtctld.CommonArg.TopoGlobalAddress, + "-topo_global_root", vtctld.CommonArg.TopoGlobalRoot, + "-cell", cell, + "-web_dir", vtctld.WebDir, + "-web_dir2", vtctld.WebDir2, + "-workflow_manager_init", + "-workflow_manager_use_election", + "-service_map", vtctld.ServiceMap, + "-backup_storage_implementation", vtctld.BackupStorageImplementation, + "-file_backup_storage_root", vtctld.FileBackupStorageRoot, + "-log_dir", vtctld.LogDir, + "-port", fmt.Sprintf("%d", vtctld.Port), + "-grpc_port", fmt.Sprintf("%d", vtctld.GrpcPort), + "-pid_file", vtctld.PidFile, + ) + vtctld.proc.Args = append(vtctld.proc.Args, extraArgs...) + + errFile, _ := os.Create(path.Join(vtctld.LogDir, "vtctld-stderr.txt")) + vtctld.proc.Stderr = errFile + + vtctld.proc.Env = append(vtctld.proc.Env, os.Environ()...) + + log.Infof("%v %v", strings.Join(vtctld.proc.Args, " ")) + + err = vtctld.proc.Start() + if err != nil { + return + } + + vtctld.exit = make(chan error) + go func() { + vtctld.exit <- vtctld.proc.Wait() + }() + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + if vtctld.IsHealthy() { + return nil + } + select { + case err := <-vtctld.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", vtctld.Name, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vtctld.Name, <-vtctld.exit) +} + +func createDirectory(dirName string, mode os.FileMode) error { + if _, err := os.Stat(dirName); os.IsNotExist(err) { + return os.Mkdir(dirName, mode) + } + return nil +} + +// IsHealthy function checks if vtctld process is up and running +func (vtctld *VtctldProcess) IsHealthy() bool { + resp, err := http.Get(vtctld.VerifyURL) + if err != nil { + return false + } + if resp.StatusCode == 200 { + return true + } + return false +} + +// TearDown shutdowns the running vtctld service +func (vtctld *VtctldProcess) TearDown() error { + if vtctld.proc == nil || vtctld.exit == nil { + return nil + } + + // Attempt graceful shutdown with SIGTERM first + vtctld.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-vtctld.exit: + vtctld.proc = nil + return err + + case <-time.After(10 * time.Second): + vtctld.proc.Process.Kill() + vtctld.proc = nil + return <-vtctld.exit + } +} + +// VtctldProcessInstance returns a VtctlProcess handle for vtctl process +// configured with the given Config. +// The process must be manually started by calling setup() +func VtctldProcessInstance(httpPort int, grpcPort int, topoPort int, hostname string, tmpDirectory string) *VtctldProcess { + vtctl := VtctlProcessInstance(topoPort, hostname) + vtctld := &VtctldProcess{ + Name: "vtctld", + Binary: "vtctld", + CommonArg: *vtctl, + WebDir: path.Join(os.Getenv("VTROOT"), "/web/vtctld"), + WebDir2: path.Join(os.Getenv("VTROOT"), "/web/vtctld2/app"), + ServiceMap: "grpc-vtctl", + BackupStorageImplementation: "file", + FileBackupStorageRoot: path.Join(os.Getenv("VTDATAROOT"), "/backups"), + LogDir: tmpDirectory, + Port: httpPort, + GrpcPort: grpcPort, + PidFile: path.Join(tmpDirectory, "vtctld.pid"), + Directory: os.Getenv("VTDATAROOT"), + } + vtctld.VerifyURL = fmt.Sprintf("http://%s:%d/debug/vars", hostname, vtctld.Port) + return vtctld +} diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go new file mode 100644 index 00000000000..f5b3655f88c --- /dev/null +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -0,0 +1,227 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "reflect" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +// VtgateProcess is a generic handle for a running vtgate . +// It can be spawned manually +type VtgateProcess struct { + Name string + Binary string + CommonArg VtctlProcess + LogDir string + FileToLogQueries string + Port int + GrpcPort int + MySQLServerPort int + MySQLServerSocketPath string + Cell string + CellsToWatch string + TabletTypesToWait string + GatewayImplementation string + ServiceMap string + PidFile string + MySQLAuthServerImpl string + Directory string + VerifyURL string + //Extra Args to be set before starting the vtgate process + ExtraArgs []string + + proc *exec.Cmd + exit chan error +} + +// Setup starts Vtgate process with required arguements +func (vtgate *VtgateProcess) Setup() (err error) { + + vtgate.proc = exec.Command( + vtgate.Binary, + "-topo_implementation", vtgate.CommonArg.TopoImplementation, + "-topo_global_server_address", vtgate.CommonArg.TopoGlobalAddress, + "-topo_global_root", vtgate.CommonArg.TopoGlobalRoot, + "-log_dir", vtgate.LogDir, + "-log_queries_to_file", vtgate.FileToLogQueries, + "-port", fmt.Sprintf("%d", vtgate.Port), + "-grpc_port", fmt.Sprintf("%d", vtgate.GrpcPort), + "-mysql_server_port", fmt.Sprintf("%d", vtgate.MySQLServerPort), + "-mysql_server_socket_path", vtgate.MySQLServerSocketPath, + "-cell", vtgate.Cell, + "-cells_to_watch", vtgate.CellsToWatch, + "-tablet_types_to_wait", vtgate.TabletTypesToWait, + "-gateway_implementation", vtgate.GatewayImplementation, + "-service_map", vtgate.ServiceMap, + "-mysql_auth_server_impl", vtgate.MySQLAuthServerImpl, + "-pid_file", vtgate.PidFile, + ) + vtgate.proc.Args = append(vtgate.proc.Args, vtgate.ExtraArgs...) + + errFile, _ := os.Create(path.Join(vtgate.LogDir, "vtgate-stderr.txt")) + vtgate.proc.Stderr = errFile + + vtgate.proc.Env = append(vtgate.proc.Env, os.Environ()...) + + log.Infof("%v %v", strings.Join(vtgate.proc.Args, " ")) + + err = vtgate.proc.Start() + if err != nil { + return + } + + vtgate.exit = make(chan error) + go func() { + vtgate.exit <- vtgate.proc.Wait() + }() + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + if vtgate.WaitForStatus() { + return nil + } + select { + case err := <-vtgate.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", vtgate.Name, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vtgate.Name, <-vtgate.exit) +} + +// WaitForStatus function checks if vtgate process is up and running +func (vtgate *VtgateProcess) WaitForStatus() bool { + resp, err := http.Get(vtgate.VerifyURL) + if err != nil { + return false + } + if resp.StatusCode == 200 { + return true + } + return false +} + +// GetStatusForTabletOfShard function gets status for a specific tablet of a shard in keyspace +func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string) bool { + resp, err := http.Get(vtgate.VerifyURL) + if err != nil { + return false + } + if resp.StatusCode == 200 { + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + if err != nil { + panic(err) + } + object := reflect.ValueOf(resultMap["HealthcheckConnections"]) + masterConnectionExist := false + if object.Kind() == reflect.Map { + for _, key := range object.MapKeys() { + if key.String() == name { + value := fmt.Sprintf("%v", object.MapIndex(key)) + return value == "1" + } + + } + } + return masterConnectionExist + } + return false +} + +// WaitForStatusOfTabletInShard function waits till status of a tablet in shard is 1 +func (vtgate *VtgateProcess) WaitForStatusOfTabletInShard(name string) error { + timeout := time.Now().Add(10 * time.Second) + for time.Now().Before(timeout) { + if vtgate.GetStatusForTabletOfShard(name) { + return nil + } + select { + case err := <-vtgate.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", vtgate.Name, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + return fmt.Errorf("wait for %s failed", name) +} + +// TearDown shuts down the running vtgate service +func (vtgate *VtgateProcess) TearDown() error { + if vtgate.proc == nil || vtgate.exit == nil { + return nil + } + // Attempt graceful shutdown with SIGTERM first + vtgate.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-vtgate.exit: + vtgate.proc = nil + return err + + case <-time.After(10 * time.Second): + vtgate.proc.Process.Kill() + vtgate.proc = nil + return <-vtgate.exit + } +} + +// VtgateProcessInstance returns a Vtgate handle for vtgate process +// configured with the given Config. +// The process must be manually started by calling setup() +func VtgateProcessInstance(port int, grpcPort int, mySQLServerPort int, cell string, cellsToWatch string, hostname string, tabletTypesToWait string, topoPort int, tmpDirectory string, extraArgs []string) *VtgateProcess { + vtctl := VtctlProcessInstance(topoPort, hostname) + vtgate := &VtgateProcess{ + Name: "vtgate", + Binary: "vtgate", + FileToLogQueries: path.Join(tmpDirectory, "/vtgate_querylog.txt"), + Directory: os.Getenv("VTDATAROOT"), + ServiceMap: "grpc-vtgateservice", + LogDir: tmpDirectory, + Port: port, + GrpcPort: grpcPort, + MySQLServerPort: mySQLServerPort, + MySQLServerSocketPath: path.Join(tmpDirectory, "mysql.sock"), + Cell: cell, + CellsToWatch: cellsToWatch, + TabletTypesToWait: tabletTypesToWait, + GatewayImplementation: "discoverygateway", + CommonArg: *vtctl, + PidFile: path.Join(tmpDirectory, "/vtgate.pid"), + MySQLAuthServerImpl: "none", + ExtraArgs: extraArgs, + } + + vtgate.VerifyURL = fmt.Sprintf("http://%s:%d/debug/vars", hostname, port) + + return vtgate +} diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go new file mode 100644 index 00000000000..292ed85a9a7 --- /dev/null +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -0,0 +1,245 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "reflect" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + + "vitess.io/vitess/go/vt/log" +) + +// VttabletProcess is a generic handle for a running vttablet . +// It can be spawned manually +type VttabletProcess struct { + Name string + Binary string + FileToLogQueries string + TabletUID int + TabletPath string + Cell string + Port int + GrpcPort int + PidFile string + Shard string + CommonArg VtctlProcess + LogDir string + TabletHostname string + Keyspace string + TabletType string + HealthCheckInterval int + BackupStorageImplementation string + FileBackupStorageRoot string + ServiceMap string + VtctldAddress string + Directory string + VerifyURL string + EnableSemiSync bool + SupportBackup bool + ServingStatus string + //Extra Args to be set before starting the vttablet process + ExtraArgs []string + + proc *exec.Cmd + exit chan error +} + +// Setup starts vtctld process with required arguements +func (vttablet *VttabletProcess) Setup() (err error) { + + vttablet.proc = exec.Command( + vttablet.Binary, + "-topo_implementation", vttablet.CommonArg.TopoImplementation, + "-topo_global_server_address", vttablet.CommonArg.TopoGlobalAddress, + "-topo_global_root", vttablet.CommonArg.TopoGlobalRoot, + "-log_queries_to_file", vttablet.FileToLogQueries, + "-tablet-path", vttablet.TabletPath, + "-port", fmt.Sprintf("%d", vttablet.Port), + "-grpc_port", fmt.Sprintf("%d", vttablet.GrpcPort), + "-pid_file", vttablet.PidFile, + "-init_shard", vttablet.Shard, + "-log_dir", vttablet.LogDir, + "-tablet_hostname", vttablet.TabletHostname, + "-init_keyspace", vttablet.Keyspace, + "-init_tablet_type", vttablet.TabletType, + "-health_check_interval", fmt.Sprintf("%ds", vttablet.HealthCheckInterval), + "-enable_replication_reporter", + "-backup_storage_implementation", vttablet.BackupStorageImplementation, + "-file_backup_storage_root", vttablet.FileBackupStorageRoot, + "-service_map", vttablet.ServiceMap, + "-vtctld_addr", vttablet.VtctldAddress, + ) + + if vttablet.SupportBackup { + vttablet.proc.Args = append(vttablet.proc.Args, "-restore_from_backup") + } + if vttablet.EnableSemiSync { + vttablet.proc.Args = append(vttablet.proc.Args, "-enable_semi_sync") + } + + vttablet.proc.Args = append(vttablet.proc.Args, vttablet.ExtraArgs...) + + errFile, _ := os.Create(path.Join(vttablet.LogDir, vttablet.TabletPath+"-vttablet-stderr.txt")) + vttablet.proc.Stderr = errFile + + vttablet.proc.Env = append(vttablet.proc.Env, os.Environ()...) + + log.Infof("%v %v", strings.Join(vttablet.proc.Args, " ")) + + err = vttablet.proc.Start() + if err != nil { + return + } + + vttablet.exit = make(chan error) + go func() { + vttablet.exit <- vttablet.proc.Wait() + }() + + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + if vttablet.WaitForStatus(vttablet.ServingStatus) { + return nil + } + select { + case err := <-vttablet.exit: + return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err) + default: + time.Sleep(300 * time.Millisecond) + } + } + + return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vttablet.Name, <-vttablet.exit) +} + +// WaitForStatus function checks if vttablet process is up and running +func (vttablet *VttabletProcess) WaitForStatus(status string) bool { + return vttablet.GetTabletStatus() == status +} + +// GetTabletStatus function checks if vttablet process is up and running +func (vttablet *VttabletProcess) GetTabletStatus() string { + resp, err := http.Get(vttablet.VerifyURL) + if err != nil { + return "" + } + if resp.StatusCode == 200 { + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + if err != nil { + panic(err) + } + status := reflect.ValueOf(resultMap["TabletStateName"]).String() + return status + } + return "" +} + +// TearDown shuts down the running vttablet service +func (vttablet *VttabletProcess) TearDown() error { + if vttablet.proc == nil { + fmt.Printf("No process found for vttablet %d", vttablet.TabletUID) + } + if vttablet.proc == nil || vttablet.exit == nil { + return nil + } + // Attempt graceful shutdown with SIGTERM first + vttablet.proc.Process.Signal(syscall.SIGTERM) + + select { + case <-vttablet.exit: + vttablet.proc = nil + return nil + + case <-time.After(10 * time.Second): + vttablet.proc.Process.Kill() + vttablet.proc = nil + return <-vttablet.exit + } +} + +// QueryTablet lets you execute query in this tablet and get the result +func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useDb bool) (*sqltypes.Result, error) { + dbParams := mysql.ConnParams{ + Uname: "vt_dba", + UnixSocket: path.Join(vttablet.Directory, "mysql.sock"), + } + if useDb { + dbParams.DbName = "vt_" + keyspace + } + ctx := context.Background() + dbConn, err := mysql.Connect(ctx, &dbParams) + if err != nil { + return nil, err + } + defer dbConn.Close() + return dbConn.ExecuteFetch(query, 1000, true) +} + +// VttabletProcessInstance returns a VttabletProcess handle for vttablet process +// configured with the given Config. +// The process must be manually started by calling setup() +func VttabletProcessInstance(port int, grpcPort int, tabletUID int, cell string, shard string, keyspace string, vtctldPort int, tabletType string, topoPort int, hostname string, tmpDirectory string, extraArgs []string, enableSemiSync bool) *VttabletProcess { + vtctl := VtctlProcessInstance(topoPort, hostname) + vttablet := &VttabletProcess{ + Name: "vttablet", + Binary: "vttablet", + FileToLogQueries: path.Join(tmpDirectory, fmt.Sprintf("/vt_%010d/querylog.txt", tabletUID)), + Directory: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID)), + TabletPath: fmt.Sprintf("%s-%010d", cell, tabletUID), + ServiceMap: "grpc-queryservice,grpc-tabletmanager,grpc-updatestream", + LogDir: tmpDirectory, + Shard: shard, + TabletHostname: hostname, + Keyspace: keyspace, + TabletType: "replica", + CommonArg: *vtctl, + HealthCheckInterval: 5, + BackupStorageImplementation: "file", + FileBackupStorageRoot: path.Join(os.Getenv("VTDATAROOT"), "/backups"), + Port: port, + GrpcPort: grpcPort, + PidFile: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/vttablet.pid", tabletUID)), + VtctldAddress: fmt.Sprintf("http://%s:%d", hostname, vtctldPort), + ExtraArgs: extraArgs, + EnableSemiSync: enableSemiSync, + SupportBackup: true, + ServingStatus: "NOT_SERVING", + } + + if tabletType == "rdonly" { + vttablet.TabletType = tabletType + } + vttablet.VerifyURL = fmt.Sprintf("http://%s:%d/debug/vars", hostname, port) + + return vttablet +} diff --git a/go/test/endtoend/clustertest/add_keyspace_test.go b/go/test/endtoend/clustertest/add_keyspace_test.go new file mode 100644 index 00000000000..aface0b1467 --- /dev/null +++ b/go/test/endtoend/clustertest/add_keyspace_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This adds sharded keyspace dynamically in this test only and test sql insert, select +*/ + +package clustertest + +import ( + "context" + "fmt" + "testing" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + testKeyspace = &cluster.Keyspace{ + Name: "kstest", + SchemaSQL: `create table vt_user ( +id bigint, +name varchar(64), +primary key (id) +) Engine=InnoDB`, + VSchema: `{ + "sharded": true, + "vindexes": { + "hash_index": { + "type": "hash" + } + }, + "tables": { + "vt_user": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + } + ] + } + } +}`, + } +) + +func TestAddKeyspace(t *testing.T) { + if err := clusterInstance.StartKeyspace(*testKeyspace, []string{"-80", "80-"}, 1, true); err != nil { + println(err.Error()) + t.Fatal(err) + } + // Restart vtgate process + _ = clusterInstance.VtgateProcess.TearDown() + _ = clusterInstance.VtgateProcess.Setup() + clusterInstance.WaitForTabletsToHealthyInVtgate() + + ctx := context.Background() + vtParams := mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + conn, err := mysql.Connect(ctx, &vtParams) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + exec(t, conn, "insert into vt_user(id, name) values(1,'name1')") + + qr := exec(t, conn, "select id, name from vt_user") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(1) VARCHAR("name1")]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } +} diff --git a/go/test/endtoend/clustertest/etcd_test.go b/go/test/endtoend/clustertest/etcd_test.go new file mode 100644 index 00000000000..cb0138b0d5e --- /dev/null +++ b/go/test/endtoend/clustertest/etcd_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustertest + +import ( + "fmt" + "testing" +) + +func TestEtcdServer(t *testing.T) { + etcdURL := fmt.Sprintf("http://%s:%d/v2/keys", clusterInstance.Hostname, clusterInstance.TopoPort) + testURL(t, etcdURL, "generic etcd url") + testURL(t, etcdURL+"/vitess/global", "vitess global key") + testURL(t, etcdURL+"/vitess/zone1", "vitess zone1 key") +} diff --git a/go/test/endtoend/clustertest/main_test.go b/go/test/endtoend/clustertest/main_test.go new file mode 100644 index 00000000000..11c48433ac3 --- /dev/null +++ b/go/test/endtoend/clustertest/main_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustertest + +import ( + "flag" + "net/http" + "os" + "testing" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceName = "commerce" + cell = "zone1" + sqlSchema = `create table product( + sku varbinary(128), + description varbinary(128), + price bigint, + primary key(sku) + ) ENGINE=InnoDB; + create table customer( + id bigint not null auto_increment, + email varchar(128), + primary key(id) + ) ENGINE=InnoDB; + create table corder( + order_id bigint not null auto_increment, + customer_id bigint, + sku varbinary(128), + price bigint, + primary key(order_id) + ) ENGINE=InnoDB;` + + vSchema = `{ + "tables": { + "product": {}, + "customer": {}, + "corder": {} + } + }` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + VSchema: vSchema, + } + err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, true) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} + +func testURL(t *testing.T, url string, testCaseName string) { + statusCode := getStatusForURL(url) + if got, want := statusCode, 200; got != want { + t.Errorf("select:\n%v want\n%v for %s", got, want, testCaseName) + } +} + +// getStatusForUrl returns the status code for the URL +func getStatusForURL(url string) int { + resp, _ := http.Get(url) + if resp != nil { + return resp.StatusCode + } + return 0 +} diff --git a/go/test/endtoend/clustertest/vtcltd_test.go b/go/test/endtoend/clustertest/vtcltd_test.go new file mode 100644 index 00000000000..97a157f3458 --- /dev/null +++ b/go/test/endtoend/clustertest/vtcltd_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +package clustertest + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + oneTableOutput = `+---+ +| a | ++---+ +| 1 | ++---+ +` +) + +func TestVtctldProcess(t *testing.T) { + url := fmt.Sprintf("http://%s:%d/api/keyspaces/", clusterInstance.Hostname, clusterInstance.VtctldHTTPPort) + testURL(t, url, "keyspace url") + + healthCheckURL := fmt.Sprintf("http://%s:%d/debug/health/", clusterInstance.Hostname, clusterInstance.VtctldHTTPPort) + testURL(t, healthCheckURL, "vtctld health check url") + + url = fmt.Sprintf("http://%s:%d/api/topodata/", clusterInstance.Hostname, clusterInstance.VtctldHTTPPort) + + testTopoDataAPI(t, url) + testListAllTablets(t) + testTabletStatus(t) + testExecuteAsDba(t) + testExecuteAsApp(t) +} + +func testTopoDataAPI(t *testing.T, url string) { + resp, err := http.Get(url) + assert.Nil(t, err) + assert.Equal(t, resp.StatusCode, 200) + + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err = json.Unmarshal(respByte, &resultMap) + assert.Nil(t, err) + + errorValue := reflect.ValueOf(resultMap["Error"]) + assert.Empty(t, errorValue.String()) + + assert.Contains(t, resultMap, "Children") + children := reflect.ValueOf(resultMap["Children"]) + childrenGot := fmt.Sprintf("%s", children) + assert.Contains(t, childrenGot, "global") + assert.Contains(t, childrenGot, clusterInstance.Cell) +} + +func testListAllTablets(t *testing.T) { + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ListAllTablets", clusterInstance.Cell) + assert.Nil(t, err) + + tablets := getAllTablets() + + tabletsFromCMD := strings.Split(result, "\n") + tabletCountFromCMD := 0 + + for _, line := range tabletsFromCMD { + if len(line) > 0 { + tabletCountFromCMD = tabletCountFromCMD + 1 + assert.Contains(t, tablets, strings.Split(line, " ")[0]) + } + } + assert.Equal(t, tabletCountFromCMD, len(tablets)) +} + +func testTabletStatus(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://%s:%d", clusterInstance.Hostname, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].HTTPPort)) + assert.Nil(t, err) + respByte, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + result := string(respByte) + println(result) + println(strings.Contains(result, "Polling health information from.")) + matched, err := regexp.Match(`Polling health information from.+MySQLReplicationLag`, []byte(result)) + assert.Nil(t, err) + assert.True(t, matched) + assert.True(t, strings.Contains(result, `Alias: 3*time.Second /* timeout */ { + assert.Equal(t, want, got) + break + } + + if got == want { + assert.Equal(t, want, got) + break + } else { + time.Sleep(300 * time.Millisecond /* interval at which to check again */) + } + } +} diff --git a/go/test/endtoend/tabletmanager/main_test.go b/go/test/endtoend/tabletmanager/main_test.go new file mode 100644 index 00000000000..2b9aaa0dc70 --- /dev/null +++ b/go/test/endtoend/tabletmanager/main_test.go @@ -0,0 +1,197 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletmanager + +import ( + "context" + "flag" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" + tabletpb "vitess.io/vitess/go/vt/proto/topodata" + tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + tmClient *tmc.Client + vtParams mysql.ConnParams + masterTabletParams mysql.ConnParams + replicaTabletParams mysql.ConnParams + masterTablet cluster.Vttablet + replicaTablet cluster.Vttablet + rdonlyTablet cluster.Vttablet + replicaUID int + masterUID int + hostname = "localhost" + keyspaceName = "ks" + shardName = "0" + keyspaceShard = "ks/" + shardName + dbName = "vt_" + keyspaceName + username = "vt_dba" + cell = "zone1" + sqlSchema = ` + create table t1( + id bigint, + value varchar(16), + primary key(id) + ) Engine=InnoDB; +` + + vSchema = ` + { + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + } + } + }` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // List of users authorized to execute vschema ddl operations + clusterInstance.VtGateExtraArgs = []string{"-vschema_ddl_authorized_users=%"} + // Set extra tablet args for lock timeout + clusterInstance.VtTabletExtraArgs = []string{ + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + } + // We do not need semiSync for this test case. + clusterInstance.EnableSemiSync = false + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + VSchema: vSchema, + } + + if err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, true); err != nil { + return 1 + } + + // Start vtgate + if err = clusterInstance.StartVtgate(); err != nil { + return 1 + } + + // Collect table paths and ports + tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets + for _, tablet := range tablets { + if tablet.Type == "master" { + masterTablet = tablet + } else if tablet.Type != "rdonly" { + replicaTablet = tablet + } else { + rdonlyTablet = tablet + } + } + + // Set mysql tablet params + masterTabletParams = mysql.ConnParams{ + Uname: username, + DbName: dbName, + UnixSocket: fmt.Sprintf(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/mysql.sock", masterTablet.TabletUID))), + } + replicaTabletParams = mysql.ConnParams{ + Uname: username, + DbName: dbName, + UnixSocket: fmt.Sprintf(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/mysql.sock", replicaTablet.TabletUID))), + } + + // create tablet manager client + tmClient = tmc.NewClient() + + return m.Run() + }() + os.Exit(exitCode) +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err) + return qr +} + +func tmcLockTables(ctx context.Context, tabletGrpcPort int) error { + vtablet := getTablet(tabletGrpcPort) + return tmClient.LockTables(ctx, vtablet) +} + +func tmcUnlockTables(ctx context.Context, tabletGrpcPort int) error { + vtablet := getTablet(tabletGrpcPort) + return tmClient.UnlockTables(ctx, vtablet) +} + +func tmcStopSlave(ctx context.Context, tabletGrpcPort int) error { + vtablet := getTablet(tabletGrpcPort) + return tmClient.StopSlave(ctx, vtablet) +} + +func tmcStartSlave(ctx context.Context, tabletGrpcPort int) error { + vtablet := getTablet(tabletGrpcPort) + return tmClient.StartSlave(ctx, vtablet) +} + +func tmcMasterPosition(ctx context.Context, tabletGrpcPort int) (string, error) { + vtablet := getTablet(tabletGrpcPort) + return tmClient.MasterPosition(ctx, vtablet) +} + +func tmcStartSlaveUntilAfter(ctx context.Context, tabletGrpcPort int, positon string, waittime time.Duration) error { + vtablet := getTablet(tabletGrpcPort) + return tmClient.StartSlaveUntilAfter(ctx, vtablet, positon, waittime) +} + +func getTablet(tabletGrpcPort int) *tabletpb.Tablet { + portMap := make(map[string]int32) + portMap["grpc"] = int32(tabletGrpcPort) + return &tabletpb.Tablet{Hostname: hostname, PortMap: portMap} +} diff --git a/go/test/endtoend/tabletmanager/qps_test.go b/go/test/endtoend/tabletmanager/qps_test.go new file mode 100644 index 00000000000..3b612ae8186 --- /dev/null +++ b/go/test/endtoend/tabletmanager/qps_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tabletmanager + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql" + querypb "vitess.io/vitess/go/vt/proto/query" +) + +func TestQPS(t *testing.T) { + ctx := context.Background() + + vtParams := mysql.ConnParams{ + Host: "localhost", + Port: clusterInstance.VtgateMySQLPort, + } + vtGateConn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer vtGateConn.Close() + + replicaConn, err := mysql.Connect(ctx, &replicaTabletParams) + require.NoError(t, err) + defer replicaConn.Close() + + // Sanity Check + exec(t, vtGateConn, "delete from t1") + exec(t, vtGateConn, "insert into t1(id, value) values(1,'a'), (2,'b')") + checkDataOnReplica(t, replicaConn, `[[VARCHAR("a")] [VARCHAR("b")]]`) + + // Test that VtTabletStreamHealth reports a QPS >0.0. + // Therefore, issue several reads first. + // NOTE: This may be potentially flaky because we'll observe a QPS >0.0 + // exactly "once" for the duration of one sampling interval (5s) and + // after that we'll see 0.0 QPS rates again. If this becomes actually + // flaky, we need to read continuously in a separate thread. + + n := 0 + for n < 15 { + n++ + // Run queries via vtGate so that they are counted. + exec(t, vtGateConn, "select * from t1") + } + + // This may take up to 5 seconds to become true because we sample the query + // counts for the rates only every 5 seconds. + + var qpsIncreased bool + timeout := time.Now().Add(12 * time.Second) + for time.Now().Before(timeout) { + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", masterTablet.Alias) + + var streamHealthResponse querypb.StreamHealthResponse + + err = json.Unmarshal([]byte(result), &streamHealthResponse) + require.NoError(t, err) + + realTimeStats := streamHealthResponse.GetRealtimeStats() + qps := realTimeStats.GetQps() + if qps > 0.0 { + qpsIncreased = true + break + } + time.Sleep(100 * time.Millisecond) + } + assert.True(t, qpsIncreased, "qps should be more that 0") +} diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go new file mode 100644 index 00000000000..bf6ed5e2e2b --- /dev/null +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -0,0 +1,303 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletmanager + +import ( + "bufio" + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// TabletReshuffle test if a vttablet can be pointed at an existing mysql +func TestTabletReshuffle(t *testing.T) { + ctx := context.Background() + + masterConn, err := mysql.Connect(ctx, &masterTabletParams) + require.NoError(t, err) + defer masterConn.Close() + + replicaConn, err := mysql.Connect(ctx, &replicaTabletParams) + require.NoError(t, err) + defer replicaConn.Close() + + // Sanity Check + exec(t, masterConn, "delete from t1") + exec(t, masterConn, "insert into t1(id, value) values(1,'a'), (2,'b')") + checkDataOnReplica(t, replicaConn, `[[VARCHAR("a")] [VARCHAR("b")]]`) + + //Create new tablet + rTablet := clusterInstance.GetVttabletInstance(0) + + //Init Tablets + err = clusterInstance.VtctlclientProcess.InitTablet(rTablet, cell, keyspaceName, hostname, shardName) + require.NoError(t, err) + + // mycnf_server_id prevents vttablet from reading the mycnf + // Pointing to masterTablet's socket file + clusterInstance.VtTabletExtraArgs = []string{ + "-lock_tables_timeout", "5s", + "-mycnf_server_id", fmt.Sprintf("%d", rTablet.TabletUID), + "-db_socket", fmt.Sprintf("%s/mysql.sock", masterTablet.VttabletProcess.Directory), + } + // SupportBackup=False prevents vttablet from trying to restore + // Start vttablet process + err = clusterInstance.StartVttablet(rTablet, "SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + sql := "select value from t1" + args := []string{ + "VtTabletExecute", + "-options", "included_fields:TYPE_ONLY", + "-json", + rTablet.Alias, + sql, + } + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) + assertExcludeFields(t, result) + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", rTablet.Alias) + assert.Error(t, err, "cannot perform backup without my.cnf") + + // Reset the VtTabletExtraArgs + clusterInstance.VtTabletExtraArgs = []string{} + killTablets(t, rTablet) +} + +func TestHealthCheck(t *testing.T) { + // Add one replica that starts not initialized + // (for the replica, we let vttablet do the InitTablet) + ctx := context.Background() + + rTablet := clusterInstance.GetVttabletInstance(0) + + // Start Mysql Processes and return connection + replicaConn, err := cluster.StartMySQLAndGetConnection(ctx, rTablet, username, clusterInstance.TmpDirectory) + assert.Nil(t, err) + defer replicaConn.Close() + + // Create database in mysql + exec(t, replicaConn, fmt.Sprintf("create database vt_%s", keyspaceName)) + + //Init Replica Tablet + err = clusterInstance.VtctlclientProcess.InitTablet(rTablet, cell, keyspaceName, hostname, shardName) + + // start vttablet process, should be in SERVING state as we already have a master + err = clusterInstance.StartVttablet(rTablet, "SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err, "error should be Nil") + + masterConn, err := mysql.Connect(ctx, &masterTabletParams) + require.NoError(t, err) + defer masterConn.Close() + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + assert.Nil(t, err) + checkHealth(t, replicaTablet.HTTPPort, false) + + // Make sure the master is still master + checkTabletType(t, masterTablet.Alias, "MASTER") + exec(t, masterConn, "stop slave") + + // stop replication, make sure we don't go unhealthy. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopSlave", rTablet.Alias) + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + assert.Nil(t, err) + + // make sure the health stream is updated + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", rTablet.Alias) + assert.Nil(t, err) + verifyStreamHealth(t, result) + + // then restart replication, make sure we stay healthy + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopSlave", rTablet.Alias) + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) + assert.Nil(t, err) + checkHealth(t, replicaTablet.HTTPPort, false) + + // now test VtTabletStreamHealth returns the right thing + result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "2", rTablet.Alias) + scanner := bufio.NewScanner(strings.NewReader(result)) + for scanner.Scan() { + // fmt.Println() // Println will add back the final '\n' + verifyStreamHealth(t, scanner.Text()) + } + + // Manual cleanup of processes + killTablets(t, rTablet) +} + +func checkHealth(t *testing.T, port int, shouldError bool) { + url := fmt.Sprintf("http://localhost:%d/healthz", port) + resp, err := http.Get(url) + assert.Nil(t, err) + if shouldError { + assert.True(t, resp.StatusCode > 400) + } else { + assert.Equal(t, 200, resp.StatusCode) + } +} + +func checkTabletType(t *testing.T, tabletAlias string, typeWant string) { + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tabletAlias) + assert.Nil(t, err) + + var tablet topodatapb.Tablet + err = json2.Unmarshal([]byte(result), &tablet) + assert.Nil(t, err) + + actualType := tablet.GetType() + got := fmt.Sprintf("%d", actualType) + + tabletType := topodatapb.TabletType_value[typeWant] + want := fmt.Sprintf("%d", tabletType) + + assert.Equal(t, want, got) +} + +func verifyStreamHealth(t *testing.T, result string) { + var streamHealthResponse querypb.StreamHealthResponse + err := json2.Unmarshal([]byte(result), &streamHealthResponse) + require.NoError(t, err) + serving := streamHealthResponse.GetServing() + UID := streamHealthResponse.GetTabletAlias().GetUid() + realTimeStats := streamHealthResponse.GetRealtimeStats() + secondsBehindMaster := realTimeStats.GetSecondsBehindMaster() + assert.True(t, serving, "Tablet should be in serving state") + assert.True(t, UID > 0, "Tablet should contain uid") + // secondsBehindMaster varies till 7200 so setting safe limit + assert.True(t, secondsBehindMaster < 10000, "Slave should not be behind master") +} + +func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) { + // This test is similar to test_health_check, but has the following differences: + // - the second tablet is an 'rdonly' and not a 'replica' + // - the second tablet will be set to 'drained' and we expect that + // - the query service won't be shutdown + + //Wait if tablet is not in service state + waitForTabletStatus(rdonlyTablet, "SERVING") + + // Check tablet health + checkHealth(t, rdonlyTablet.HTTPPort, false) + assert.Equal(t, "SERVING", rdonlyTablet.VttabletProcess.GetTabletStatus()) + + // Change from rdonly to drained and stop replication. (These + // actions are similar to the SplitClone vtworker command + // implementation.) The tablet will stay healthy, and the + // query service is still running. + err := clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeSlaveType", rdonlyTablet.Alias, "drained") + assert.Nil(t, err) + // Trying to drain the same tablet again, should error + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeSlaveType", rdonlyTablet.Alias, "drained") + assert.Error(t, err, "already drained") + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopSlave", rdonlyTablet.Alias) + assert.Nil(t, err) + // Trigger healthcheck explicitly to avoid waiting for the next interval. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) + assert.Nil(t, err) + + checkTabletType(t, rdonlyTablet.Alias, "DRAINED") + + // Query service is still running. + waitForTabletStatus(rdonlyTablet, "SERVING") + + // Restart replication. Tablet will become healthy again. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeSlaveType", rdonlyTablet.Alias, "rdonly") + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartSlave", rdonlyTablet.Alias) + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rdonlyTablet.Alias) + assert.Nil(t, err) + checkHealth(t, rdonlyTablet.HTTPPort, false) +} + +func waitForTabletStatus(tablet cluster.Vttablet, status string) error { + timeout := time.Now().Add(10 * time.Second) + for time.Now().Before(timeout) { + if tablet.VttabletProcess.WaitForStatus(status) { + return nil + } + time.Sleep(300 * time.Millisecond) + } + return fmt.Errorf("Tablet status is not %s ", status) +} + +func TestIgnoreHealthError(t *testing.T) { + ctx := context.Background() + mTablet := clusterInstance.GetVttabletInstance(masterUID) + + //Init Tablets + err := clusterInstance.VtctlclientProcess.InitTablet(mTablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // Start Mysql Processes + masterConn, err := cluster.StartMySQLAndGetConnection(ctx, mTablet, username, clusterInstance.TmpDirectory) + defer masterConn.Close() + assert.Nil(t, err) + + mTablet.MysqlctlProcess.Stop() + // Clean dir for mysql files + mTablet.MysqlctlProcess.CleanupFiles(mTablet.TabletUID) + + // Start Vttablet, it should be NOT_SERVING state as mysql is stopped + err = clusterInstance.StartVttablet(mTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // Force it healthy. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("IgnoreHealthError", mTablet.Alias, ".*no slave status.*") + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", mTablet.Alias) + assert.Nil(t, err) + waitForTabletStatus(*mTablet, "SERVING") + + // Turn off the force-healthy. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("IgnoreHealthError", mTablet.Alias, "") + assert.Nil(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", mTablet.Alias) + assert.Nil(t, err) + waitForTabletStatus(*mTablet, "NOT_SERVING") + checkHealth(t, mTablet.HTTPPort, true) + + // Tear down custom processes + killTablets(t, mTablet) +} + +func killTablets(t *testing.T, tablets ...*cluster.Vttablet) { + for _, tablet := range tablets { + //Stop Mysqld + tablet.MysqlctlProcess.Stop() + + //Tear down Tablet + tablet.VttabletProcess.TearDown() + + } +} \ No newline at end of file diff --git a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go new file mode 100644 index 00000000000..0442293cbf0 --- /dev/null +++ b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tabletmanager + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +func TestFallbackSecurityPolicy(t *testing.T) { + ctx := context.Background() + mTablet := clusterInstance.GetVttabletInstance(masterUID) + + //Init Tablets + err := clusterInstance.VtctlclientProcess.InitTablet(mTablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // Start Mysql Processes + err = cluster.StartMySQL(ctx, mTablet, username, clusterInstance.TmpDirectory) + assert.Nil(t, err) + + // Requesting an unregistered security_policy should fallback to deny-all. + clusterInstance.VtTabletExtraArgs = []string{"-security_policy", "bogus"} + err = clusterInstance.StartVttablet(mTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // It should deny ADMIN role. + url := fmt.Sprintf("http://localhost:%d/streamqueryz/terminate", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // It should deny MONITORING role. + url = fmt.Sprintf("http://localhost:%d/debug/health", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // It should deny DEBUGGING role. + url = fmt.Sprintf("http://localhost:%d/queryz", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // Reset the VtTabletExtraArgs + clusterInstance.VtTabletExtraArgs = []string{} + // Tear down custom processes + killTablets(t, mTablet) +} + +func assertNotAllowedURLTest(t *testing.T, url string) { + resp, err := http.Get(url) + assert.Nil(t, err) + + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + assert.True(t, resp.StatusCode > 400) + assert.Contains(t, string(body), "Access denied: not allowed") +} + +func assertAllowedURLTest(t *testing.T, url string) { + resp, err := http.Get(url) + assert.Nil(t, err) + + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + assert.NotContains(t, string(body), "Access denied: not allowed") +} + +func TestDenyAllSecurityPolicy(t *testing.T) { + ctx := context.Background() + mTablet := clusterInstance.GetVttabletInstance(masterUID) + + //Init Tablets + err := clusterInstance.VtctlclientProcess.InitTablet(mTablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // Start Mysql Processes + err = cluster.StartMySQL(ctx, mTablet, username, clusterInstance.TmpDirectory) + assert.Nil(t, err) + + // Requesting a deny-all security_policy. + clusterInstance.VtTabletExtraArgs = []string{"-security_policy", "deny-all"} + err = clusterInstance.StartVttablet(mTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // It should deny ADMIN role. + url := fmt.Sprintf("http://localhost:%d/streamqueryz/terminate", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // It should deny MONITORING role. + url = fmt.Sprintf("http://localhost:%d/debug/health", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // It should deny DEBUGGING role. + url = fmt.Sprintf("http://localhost:%d/queryz", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // Reset the VtTabletExtraArgs + clusterInstance.VtTabletExtraArgs = []string{} + // Tear down custom processes + killTablets(t, mTablet) +} + +func TestReadOnlySecurityPolicy(t *testing.T) { + ctx := context.Background() + mTablet := clusterInstance.GetVttabletInstance(0) + + //Init Tablets + err := clusterInstance.VtctlclientProcess.InitTablet(mTablet, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // Start Mysql Processes + err = cluster.StartMySQL(ctx, mTablet, username, clusterInstance.TmpDirectory) + assert.Nil(t, err) + + // Requesting a read-only security_policy. + clusterInstance.VtTabletExtraArgs = []string{"-security_policy", "read-only"} + err = clusterInstance.StartVttablet(mTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) + assert.Nil(t, err) + + // It should deny ADMIN role. + url := fmt.Sprintf("http://localhost:%d/streamqueryz/terminate", mTablet.HTTPPort) + assertNotAllowedURLTest(t, url) + + // It should deny MONITORING role. + url = fmt.Sprintf("http://localhost:%d/debug/health", mTablet.HTTPPort) + assertAllowedURLTest(t, url) + + // It should deny DEBUGGING role. + url = fmt.Sprintf("http://localhost:%d/queryz", mTablet.HTTPPort) + assertAllowedURLTest(t, url) + + // Reset the VtTabletExtraArgs + clusterInstance.VtTabletExtraArgs = []string{} + // Tear down custom processes + killTablets(t, mTablet) +} diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go new file mode 100644 index 00000000000..87acd6902c4 --- /dev/null +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" +) + +func TestAggregateTypes(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + exec(t, conn, "insert into aggr_test(id, val1, val2) values(1,'a',1), (2,'A',1), (3,'b',1), (4,'c',3), (5,'c',4)") + exec(t, conn, "insert into aggr_test(id, val1, val2) values(6,'d',null), (7,'e',null), (8,'E',1)") + + qr := exec(t, conn, "select val1, count(distinct val2), count(*) from aggr_test group by val1") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + qr = exec(t, conn, "select val1, sum(distinct val2), sum(val2) from aggr_test group by val1") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("a") DECIMAL(1) DECIMAL(2)] [VARCHAR("b") DECIMAL(1) DECIMAL(1)] [VARCHAR("c") DECIMAL(7) DECIMAL(7)] [VARCHAR("d") NULL NULL] [VARCHAR("e") DECIMAL(1) DECIMAL(1)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + qr = exec(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)] [VARCHAR("d") INT64(0) INT64(1)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + qr = exec(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1 limit 4") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } +} diff --git a/go/test/endtoend/vtgate/buffer/buffer_test.go b/go/test/endtoend/vtgate/buffer/buffer_test.go new file mode 100644 index 00000000000..4518f56b079 --- /dev/null +++ b/go/test/endtoend/vtgate/buffer/buffer_test.go @@ -0,0 +1,435 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Test the vtgate master buffer. + +During a master failover, vtgate should automatically buffer (stall) requests +for a configured time and retry them after the failover is over. + +The test reproduces such a scenario as follows: +- run two threads, the first thread continuously executes a critical read and the second executes a write (UPDATE) +- vtctl PlannedReparentShard runs a master failover +- both threads should not see any error during the failover +*/ + +package buffer + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "os" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" + tabletpb "vitess.io/vitess/go/vt/proto/topodata" + tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceUnshardedName = "ks1" + cell = "zone1" + hostname = "localhost" + sqlSchema = ` + create table buffer( + id BIGINT NOT NULL, + msg VARCHAR(64) NOT NULL, + PRIMARY KEY (id) + ) Engine=InnoDB;` + wg = &sync.WaitGroup{} + tmClient = tmc.NewClient() +) + +const ( + criticalReadRowID = 1 + updateRowID = 2 + demoteMasterQuery = "SET GLOBAL read_only = ON;FLUSH TABLES WITH READ LOCK;UNLOCK TABLES;" + disableSemiSyncMasterQuery = "SET GLOBAL rpl_semi_sync_master_enabled = 0" + enableSemiSyncMasterQuery = "SET GLOBAL rpl_semi_sync_master_enabled = 1" + masterPositionQuery = "SELECT @@GLOBAL.gtid_executed;" + promoteSlaveQuery = "STOP SLAVE;RESET SLAVE ALL;SET GLOBAL read_only = OFF;" +) + +//threadParams is set of params passed into read and write threads +type threadParams struct { + writable bool + quit bool + rpcs int // Number of queries successfully executed. + errors int // Number of failed queries. + waitForNotification chan bool // Channel used to notify the main thread that this thread executed + notifyLock sync.Mutex // notifyLock guards the two fields notifyAfterNSuccessfulRpcs/rpcsSoFar. + notifyAfterNSuccessfulRpcs int // If 0, notifications are disabled + rpcsSoFar int // Number of RPCs at the time a notification was requested + i int // + commitErrors int + executeFunction func(c *threadParams, conn *mysql.Conn) error // Implement the method for read/update. +} + +// Thread which constantly executes a query on vtgate. +func (c *threadParams) threadRun() { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + if err != nil { + println(err.Error()) + } + defer conn.Close() + for !c.quit { + err = c.executeFunction(c, conn) + if err != nil { + c.errors++ + println(err.Error()) + } + c.rpcs++ + // If notifications are requested, check if we already executed the + // required number of successful RPCs. + // Use >= instead of == because we can miss the exact point due to + // slow thread scheduling. + c.notifyLock.Lock() + if c.notifyAfterNSuccessfulRpcs != 0 && c.rpcs >= (c.notifyAfterNSuccessfulRpcs+c.rpcsSoFar) { + c.waitForNotification <- true + c.notifyAfterNSuccessfulRpcs = 0 + } + c.notifyLock.Unlock() + // Wait 10ms seconds between two attempts. + time.Sleep(10 * time.Millisecond) + } + wg.Done() +} + +func (c *threadParams) setNotifyAfterNSuccessfulRpcs(n int) { + c.notifyLock.Lock() + c.notifyAfterNSuccessfulRpcs = n + c.rpcsSoFar = c.rpcs + c.notifyLock.Unlock() +} + +func (c *threadParams) stop() { + c.quit = true +} + +func readExecute(c *threadParams, conn *mysql.Conn) error { + _, err := conn.ExecuteFetch(fmt.Sprintf("SELECT * FROM buffer WHERE id = %d", criticalReadRowID), 1000, true) + return err +} + +func updateExecute(c *threadParams, conn *mysql.Conn) error { + attempts := c.i + // Value used in next UPDATE query. Increased after every query. + c.i++ + conn.ExecuteFetch("begin", 1000, true) + + _, err := conn.ExecuteFetch(fmt.Sprintf("UPDATE buffer SET msg='update %d' WHERE id = %d", attempts, updateRowID), 1000, true) + + // Sleep between [0, 1] seconds to prolong the time the transaction is in + // flight. This is more realistic because applications are going to keep + // their transactions open for longer as well. + time.Sleep(time.Duration(rand.Int31n(1000)) * time.Millisecond) + + if err == nil { + fmt.Printf("update %d affected", attempts) + _, err = conn.ExecuteFetch("commit", 1000, true) + if err != nil { + _, errRollback := conn.ExecuteFetch("rollback", 1000, true) + if errRollback != nil { + fmt.Print("Error in rollback", errRollback.Error()) + } + c.commitErrors++ + if c.commitErrors > 1 { + return err + } + fmt.Printf("UPDATE %d failed during ROLLBACK. This is okay once because we do not support buffering it. err: %s", attempts, err.Error()) + } + } + if err != nil { + _, errRollback := conn.ExecuteFetch("rollback", 1000, true) + if errRollback != nil { + fmt.Print("Error in rollback", errRollback.Error()) + } + c.commitErrors++ + if c.commitErrors > 1 { + return err + } + fmt.Printf("UPDATE %d failed during COMMIT with err: %s.This is okay once because we do not support buffering it.", attempts, err.Error()) + } + return nil +} + +func createCluster() (*cluster.LocalProcessCluster, int) { + clusterInstance = cluster.NewCluster(cell, hostname) + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return nil, 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceUnshardedName, + SchemaSQL: sqlSchema, + } + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false); err != nil { + return nil, 1 + } + + clusterInstance.VtGateExtraArgs = []string{ + "-enable_buffer", + // Long timeout in case failover is slow. + "-buffer_window", "10m", + "-buffer_max_failover_duration", "10m", + "-buffer_min_time_between_failovers", "20m"} + + // Start vtgate + if err := clusterInstance.StartVtgate(); err != nil { + return nil, 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + rand.Seed(time.Now().UnixNano()) + return clusterInstance, 0 +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err) + return qr +} + +func TestBufferInternalReparenting(t *testing.T) { + testBufferBase(t, false) +} + +func TestBufferExternalReparenting(t *testing.T) { + testBufferBase(t, true) +} + +func testBufferBase(t *testing.T, isExternalParent bool) { + clusterInstance, exitCode := createCluster() + if exitCode != 0 { + os.Exit(exitCode) + } + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + // Insert two rows for the later threads (critical read, update). + exec(t, conn, fmt.Sprintf("INSERT INTO buffer (id, msg) VALUES (%d, %s)", criticalReadRowID, "'critical read'")) + exec(t, conn, fmt.Sprintf("INSERT INTO buffer (id, msg) VALUES (%d, %s)", updateRowID, "'update'")) + + //Start both threads. + readThreadInstance := &threadParams{writable: false, quit: false, rpcs: 0, errors: 0, notifyAfterNSuccessfulRpcs: 0, rpcsSoFar: 0, executeFunction: readExecute, waitForNotification: make(chan bool)} + wg.Add(1) + go readThreadInstance.threadRun() + updateThreadInstance := &threadParams{writable: false, quit: false, rpcs: 0, errors: 0, notifyAfterNSuccessfulRpcs: 0, rpcsSoFar: 0, executeFunction: updateExecute, i: 1, commitErrors: 0, waitForNotification: make(chan bool)} + wg.Add(1) + go updateThreadInstance.threadRun() + + // Verify they got at least 2 RPCs through. + readThreadInstance.setNotifyAfterNSuccessfulRpcs(2) + updateThreadInstance.setNotifyAfterNSuccessfulRpcs(2) + + <-readThreadInstance.waitForNotification + <-updateThreadInstance.waitForNotification + + // Execute the failover. + readThreadInstance.setNotifyAfterNSuccessfulRpcs(10) + updateThreadInstance.setNotifyAfterNSuccessfulRpcs(10) + + if isExternalParent { + externalReparenting(ctx, t, clusterInstance) + } else { + //reparent call + clusterInstance.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", "-keyspace_shard", + fmt.Sprintf("%s/%s", keyspaceUnshardedName, "0"), + "-new_master", clusterInstance.Keyspaces[0].Shards[0].Vttablets[1].Alias) + } + + <-readThreadInstance.waitForNotification + <-updateThreadInstance.waitForNotification + + // Stop threads + readThreadInstance.stop() + updateThreadInstance.stop() + + // Both threads must not see any error + assert.Equal(t, 0, readThreadInstance.errors) + assert.Equal(t, 0, updateThreadInstance.errors) + + //At least one thread should have been buffered. + //This may fail if a failover is too fast. Add retries then. + resp, err := http.Get(clusterInstance.VtgateProcess.VerifyURL) + require.NoError(t, err) + label := fmt.Sprintf("%s.%s", keyspaceUnshardedName, "0") + inFlightMax := 0 + masterPromotedCount := 0 + durationMs := 0 + bufferingStops := 0 + if resp.StatusCode == 200 { + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + if err != nil { + panic(err) + } + inFlightMax = getVarFromVtgate(t, label, "BufferLastRequestsInFlightMax", resultMap) + masterPromotedCount = getVarFromVtgate(t, label, "HealthcheckMasterPromoted", resultMap) + durationMs = getVarFromVtgate(t, label, "BufferFailoverDurationSumMs", resultMap) + bufferingStops = getVarFromVtgate(t, "NewMasterSeen", "BufferStops", resultMap) + } + if inFlightMax == 0 { + // Missed buffering is okay when we observed the failover during the + // COMMIT (which cannot trigger the buffering). + assert.Greater(t, updateThreadInstance.commitErrors, 0, "No buffering took place and the update thread saw no error during COMMIT. But one of it must happen.") + } else { + assert.Greater(t, inFlightMax, 0) + } + + // There was a failover and the HealthCheck module must have seen it. + if masterPromotedCount > 0 { + assert.Greater(t, masterPromotedCount, 0) + } + + if durationMs > 0 { + // Number of buffering stops must be equal to the number of seen failovers. + assert.Equal(t, masterPromotedCount, bufferingStops) + } + wg.Wait() + clusterInstance.Teardown() +} + +func getVarFromVtgate(t *testing.T, label string, param string, resultMap map[string]interface{}) int { + paramVal := 0 + var err error + object := reflect.ValueOf(resultMap[param]) + if object.Kind() == reflect.Map { + for _, key := range object.MapKeys() { + if strings.Contains(key.String(), label) { + v := object.MapIndex(key) + s := fmt.Sprintf("%v", v.Interface()) + paramVal, err = strconv.Atoi(s) + require.NoError(t, err) + } + } + } + return paramVal +} + +func externalReparenting(ctx context.Context, t *testing.T, clusterInstance *cluster.LocalProcessCluster) { + start := time.Now() + + // Demote master Query + master := clusterInstance.Keyspaces[0].Shards[0].Vttablets[0] + replica := clusterInstance.Keyspaces[0].Shards[0].Vttablets[1] + oldMaster := master + newMaster := replica + master.VttabletProcess.QueryTablet(demoteMasterQuery, keyspaceUnshardedName, true) + if master.VttabletProcess.EnableSemiSync { + master.VttabletProcess.QueryTablet(disableSemiSyncMasterQuery, keyspaceUnshardedName, true) + } + + // Wait for replica to catch up to master. + waitForReplicationPos(ctx, t, &master, &replica, 60.0) + + duration := time.Since(start) + minUnavailabilityInS := 1.0 + if duration.Seconds() < minUnavailabilityInS { + w := minUnavailabilityInS - duration.Seconds() + fmt.Printf("Waiting for %.1f seconds because the failover was too fast (took only %.3f seconds)", w, duration.Seconds()) + time.Sleep(time.Duration(w) * time.Second) + } + + // Promote replica to new master. + replica.VttabletProcess.QueryTablet(promoteSlaveQuery, keyspaceUnshardedName, true) + + if replica.VttabletProcess.EnableSemiSync { + replica.VttabletProcess.QueryTablet(enableSemiSyncMasterQuery, keyspaceUnshardedName, true) + } + + // Configure old master to replicate from new master. + _, gtID := getMasterPosition(ctx, t, &newMaster) + + // Use 'localhost' as hostname because Travis CI worker hostnames + // are too long for MySQL replication. + changeMasterCommands := fmt.Sprintf("RESET SLAVE;SET GLOBAL gtid_slave_pos = '%s';CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d ,MASTER_USER='vt_repl', MASTER_USE_GTID = slave_pos;START SLAVE;", gtID, "localhost", newMaster.MySQLPort) + oldMaster.VttabletProcess.QueryTablet(changeMasterCommands, keyspaceUnshardedName, true) + + // Notify the new vttablet master about the reparent. + clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", newMaster.Alias) +} + +func waitForReplicationPos(ctx context.Context, t *testing.T, tabletA *cluster.Vttablet, tabletB *cluster.Vttablet, timeout float64) { + replicationPosA, _ := getMasterPosition(ctx, t, tabletA) + for { + replicationPosB, _ := getMasterPosition(ctx, t, tabletB) + if positionAtLeast(t, tabletA, replicationPosB, replicationPosA) { + break + } + msg := fmt.Sprintf("%s's replication position to catch up to %s's;currently at: %s, waiting to catch up to: %s", tabletB.Alias, tabletA.Alias, replicationPosB, replicationPosA) + waitStep(t, msg, timeout, 0.01) + } +} + +func getMasterPosition(ctx context.Context, t *testing.T, tablet *cluster.Vttablet) (string, string) { + vtablet := getTablet(tablet.GrpcPort) + newPos, err := tmClient.MasterPosition(ctx, vtablet) + require.NoError(t, err) + gtID := strings.SplitAfter(newPos, "/")[1] + return newPos, gtID +} + +func positionAtLeast(t *testing.T, tablet *cluster.Vttablet, a string, b string) bool { + isAtleast := false + val, err := tablet.MysqlctlProcess.ExecuteCommandWithOutput("position", "at_least", a, b) + require.NoError(t, err) + if strings.Contains(val, "true") { + isAtleast = true + } + return isAtleast +} + +func waitStep(t *testing.T, msg string, timeout float64, sleepTime float64) float64 { + timeout = timeout - sleepTime + if timeout < 0.0 { + t.Errorf("timeout waiting for condition '%s'", msg) + } + time.Sleep(time.Duration(sleepTime) * time.Second) + return timeout +} + +func getTablet(tabletGrpcPort int) *tabletpb.Tablet { + portMap := make(map[string]int32) + portMap["grpc"] = int32(tabletGrpcPort) + return &tabletpb.Tablet{Hostname: hostname, PortMap: portMap} +} diff --git a/go/test/endtoend/vtgate/lookup_test.go b/go/test/endtoend/vtgate/lookup_test.go new file mode 100644 index 00000000000..8c5c2c5a2c9 --- /dev/null +++ b/go/test/endtoend/vtgate/lookup_test.go @@ -0,0 +1,258 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" +) + +func TestConsistentLookup(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + // conn2 is for queries that target shards. + conn2, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn2.Close() + + // Simple insert. + exec(t, conn, "begin") + exec(t, conn, "insert into t1(id1, id2) values(1, 4)") + exec(t, conn, "commit") + qr := exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4) VARBINARY(\"\\x16k@\\xb4J\\xbaK\\xd6\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Inserting again should fail. + exec(t, conn, "begin") + _, err = conn.ExecuteFetch("insert into t1(id1, id2) values(1, 4)", 1000, false) + exec(t, conn, "rollback") + want := "duplicate entry" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("second insert: %v, must contain %s", err, want) + } + + // Simple delete. + exec(t, conn, "begin") + exec(t, conn, "delete from t1 where id1=1") + exec(t, conn, "commit") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Autocommit insert. + exec(t, conn, "insert into t1(id1, id2) values(1, 4)") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select id2 from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + // Autocommit delete. + exec(t, conn, "delete from t1 where id1=1") + + // Dangling row pointing to existing keyspace id. + exec(t, conn, "insert into t1(id1, id2) values(1, 4)") + // Delete the main row only. + exec(t, conn2, "use `ks:-80`") + exec(t, conn2, "delete from t1 where id1=1") + // Verify the lookup row is still there. + qr = exec(t, conn, "select id2 from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + // Insert should still succeed. + exec(t, conn, "begin") + exec(t, conn, "insert into t1(id1, id2) values(1, 4)") + exec(t, conn, "commit") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + // Lookup row should be unchanged. + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4) VARBINARY(\"\\x16k@\\xb4J\\xbaK\\xd6\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Dangling row not pointing to existing keyspace id. + exec(t, conn2, "use `ks:-80`") + exec(t, conn2, "delete from t1 where id1=1") + // Update the lookup row with bogus keyspace id. + exec(t, conn, "update t1_id2_idx set keyspace_id='aaa' where id2=4") + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4) VARBINARY(\"aaa\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + // Insert should still succeed. + exec(t, conn, "begin") + exec(t, conn, "insert into t1(id1, id2) values(1, 4)") + exec(t, conn, "commit") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + // lookup row must be updated. + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4) VARBINARY(\"\\x16k@\\xb4J\\xbaK\\xd6\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Update, but don't change anything. This should not deadlock. + exec(t, conn, "begin") + exec(t, conn, "update t1 set id2=4 where id1=1") + exec(t, conn, "commit") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(4) VARBINARY(\"\\x16k@\\xb4J\\xbaK\\xd6\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Update, and change the lookup value. This should change main and lookup rows. + exec(t, conn, "begin") + exec(t, conn, "update t1 set id2=5 where id1=1") + exec(t, conn, "commit") + qr = exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(5)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select * from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(5) VARBINARY(\"\\x16k@\\xb4J\\xbaK\\xd6\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + exec(t, conn, "delete from t1 where id1=1") +} + +func TestConsistentLookupMultiInsert(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + // conn2 is for queries that target shards. + conn2, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn2.Close() + + exec(t, conn, "begin") + exec(t, conn, "insert into t1(id1, id2) values(1,4), (2,5)") + exec(t, conn, "commit") + qr := exec(t, conn, "select * from t1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(4)] [INT64(2) INT64(5)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select count(*) from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(2)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Delete one row but leave its lookup dangling. + exec(t, conn2, "use `ks:-80`") + exec(t, conn2, "delete from t1 where id1=1") + // Insert a bogus lookup row. + exec(t, conn, "insert into t1_id2_idx(id2, keyspace_id) values(6, 'aaa')") + // Insert 3 rows: + // first row will insert without changing lookup. + // second will insert and change lookup. + // third will be a fresh insert for main and lookup. + exec(t, conn, "begin") + exec(t, conn, "insert into t1(id1, id2) values(1,2), (3,6), (4,7)") + exec(t, conn, "commit") + qr = exec(t, conn, "select id1, id2 from t1 order by id1") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(1) INT64(2)] [INT64(2) INT64(5)] [INT64(3) INT64(6)] [INT64(4) INT64(7)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select * from t1_id2_idx where id2=6") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(6) VARBINARY(\"N\\xb1\\x90ɢ\\xfa\\x16\\x9c\")]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select count(*) from t1_id2_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(5)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + exec(t, conn, "delete from t1 where id1=1") + exec(t, conn, "delete from t1 where id1=2") + exec(t, conn, "delete from t1 where id1=3") + exec(t, conn, "delete from t1 where id1=4") + exec(t, conn, "delete from t1_id2_idx where id2=4") +} + +func TestHashLookupMultiInsertIgnore(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + // conn2 is for queries that target shards. + conn2, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn2.Close() + + // DB should start out clean + qr := exec(t, conn, "select count(*) from t2_id4_idx") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(0)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select count(*) from t2") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(0)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Try inserting a bunch of ids at once + exec(t, conn, "begin") + exec(t, conn, "insert ignore into t2(id3, id4) values(50,60), (30,40), (10,20)") + exec(t, conn, "commit") + + // Verify + qr = exec(t, conn, "select id3, id4 from t2 order by id3") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(10) INT64(20)] [INT64(30) INT64(40)] [INT64(50) INT64(60)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + qr = exec(t, conn, "select id3, id4 from t2_id4_idx order by id3") + if got, want := fmt.Sprintf("%v", qr.Rows), "[[INT64(10) INT64(20)] [INT64(30) INT64(40)] [INT64(50) INT64(60)]]"; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err) + return qr +} diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go new file mode 100644 index 00000000000..9d506ee422b --- /dev/null +++ b/go/test/endtoend/vtgate/main_test.go @@ -0,0 +1,203 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + "flag" + "os" + "testing" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + KeyspaceName = "ks" + Cell = "test" + SchemaSQL = `create table t1( + id1 bigint, + id2 bigint, + primary key(id1) +) Engine=InnoDB; + +create table t1_id2_idx( + id2 bigint, + keyspace_id varbinary(10), + primary key(id2) +) Engine=InnoDB; + +create table vstream_test( + id bigint, + val bigint, + primary key(id) +) Engine=InnoDB; + +create table aggr_test( + id bigint, + val1 varchar(16), + val2 bigint, + primary key(id) +) Engine=InnoDB; + +create table t2( + id3 bigint, + id4 bigint, + primary key(id3) +) Engine=InnoDB; + +create table t2_id4_idx( + id bigint not null auto_increment, + id4 bigint, + id3 bigint, + primary key(id), + key idx_id4(id4) +) Engine=InnoDB; +` + + VSchema = ` + { + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + }, + "t1_id2_vdx": { + "type": "consistent_lookup_unique", + "params": { + "table": "t1_id2_idx", + "from": "id2", + "to": "keyspace_id" + }, + "owner": "t1" + }, + "t2_id4_idx": { + "type": "lookup_hash", + "params": { + "table": "t2_id4_idx", + "from": "id4", + "to": "id3", + "autocommit": "true" + }, + "owner": "t2" + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "column": "id1", + "name": "hash" + }, + { + "column": "id2", + "name": "t1_id2_vdx" + } + ] + }, + "t1_id2_idx": { + "column_vindexes": [ + { + "column": "id2", + "name": "hash" + } + ] + }, + "t2": { + "column_vindexes": [ + { + "column": "id3", + "name": "hash" + }, + { + "column": "id4", + "name": "t2_id4_idx" + } + ] + }, + "t2_id4_idx": { + "column_vindexes": [ + { + "column": "id4", + "name": "hash" + } + ] + }, + "vstream_test": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + }, + "aggr_test": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ], + "columns": [ + { + "name": "val1", + "type": "VARCHAR" + } + ] + } + } +}` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(Cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: KeyspaceName, + SchemaSQL: SchemaSQL, + VSchema: VSchema, + } + err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, true) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} diff --git a/go/test/endtoend/vtgate/schema/schema_test.go b/go/test/endtoend/vtgate/schema/schema_test.go new file mode 100644 index 00000000000..7ecc76c68b7 --- /dev/null +++ b/go/test/endtoend/vtgate/schema/schema_test.go @@ -0,0 +1,288 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "reflect" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + hostname = "localhost" + keyspaceName = "ks" + cell = "zone1" + schemaChangeDirectory = "" + totalTableCount = 4 + createTable = ` + CREATE TABLE %s ( + id BIGINT(20) not NULL, + msg varchar(64), + PRIMARY KEY (id) + ) ENGINE=InnoDB;` + alterTable = ` + ALTER TABLE %s + ADD COLUMN new_id bigint(20) NOT NULL AUTO_INCREMENT FIRST, + DROP PRIMARY KEY, + ADD PRIMARY KEY (new_id), + ADD INDEX idx_column(%s)` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitcode, err := func() (int, error) { + clusterInstance = &cluster.LocalProcessCluster{Cell: cell, Hostname: hostname} + schemaChangeDirectory = path.Join("/tmp", fmt.Sprintf("schema_change_dir_%d", clusterInstance.GetAndReserveTabletUID())) + defer os.RemoveAll(schemaChangeDirectory) + defer clusterInstance.Teardown() + + if _, err := os.Stat(schemaChangeDirectory); os.IsNotExist(err) { + _ = os.Mkdir(schemaChangeDirectory, 0700) + } + + clusterInstance.VtctldExtraArgs = []string{ + "-schema_change_dir", schemaChangeDirectory, + "-schema_change_controller", "local", + "-schema_change_check_interval", "1"} + + if err := clusterInstance.StartTopo(); err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 2, true); err != nil { + return 1, err + } + if err := clusterInstance.StartKeyspace(*keyspace, []string{"1"}, 1, false); err != nil { + return 1, err + } + return m.Run(), nil + }() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } else { + os.Exit(exitcode) + } + +} + +func TestSchemaChange(t *testing.T) { + testWithInitialSchema(t) + testWithAlterSchema(t) + testWithDropCreateSchema(t) + testSchemaChangePreflightErrorPartially(t) + testDropNonExistentTables(t) + testCopySchemaShards(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, 2) + testCopySchemaShards(t, fmt.Sprintf("%s/0", keyspaceName), 3) + testCopySchemaShardWithDifferentDB(t, 4) + testWithAutoSchemaFromChangeDir(t) +} + +func testWithInitialSchema(t *testing.T) { + // Create 4 tables + var sqlQuery = "" + for i := 0; i < totalTableCount; i++ { + sqlQuery = fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", i)) + err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery) + assert.Nil(t, err) + + } + + // Check if 4 tables are created + checkTables(t, totalTableCount) + checkTables(t, totalTableCount) + + // Also match the vschema for those tablets + matchSchema(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, clusterInstance.Keyspaces[0].Shards[1].Vttablets[0].VttabletProcess.TabletPath) +} + +// testWithAlterSchema if we alter schema and then apply, the resultant schema should match across shards +func testWithAlterSchema(t *testing.T) { + sqlQuery := fmt.Sprintf(alterTable, fmt.Sprintf("vt_select_test_%02d", 3), "msg") + err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery) + assert.Nil(t, err) + matchSchema(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, clusterInstance.Keyspaces[0].Shards[1].Vttablets[0].VttabletProcess.TabletPath) +} + +// testWithDropCreateSchema , we should be able to drop and create same schema +//Tests that a DROP and CREATE table will pass PreflightSchema check. +// +//PreflightSchema checks each SQL statement separately. When doing so, it must +//consider previous statements within the same ApplySchema command. For +//example, a CREATE after DROP must not fail: When CREATE is checked, DROP +//must have been executed first. +//See: https://github.com/vitessio/vitess/issues/1731#issuecomment-222914389 +func testWithDropCreateSchema(t *testing.T) { + dropCreateTable := fmt.Sprintf("DROP TABLE vt_select_test_%02d ;", 2) + fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", 2)) + err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, dropCreateTable) + assert.Nil(t, err) + checkTables(t, totalTableCount) +} + +// testWithAutoSchemaFromChangeDir on putting sql file to schema change directory, it should apply that sql to all shards +func testWithAutoSchemaFromChangeDir(t *testing.T) { + _ = os.Mkdir(path.Join(schemaChangeDirectory, keyspaceName), 0700) + _ = os.Mkdir(path.Join(schemaChangeDirectory, keyspaceName, "input"), 0700) + sqlFile := path.Join(schemaChangeDirectory, keyspaceName, "input/create_test_table_x.sql") + err := ioutil.WriteFile(sqlFile, []byte("create table test_table_x (id int)"), 0644) + assert.Nil(t, err) + timeout := time.Now().Add(10 * time.Second) + matchFoundAfterAutoSchemaApply := false + for time.Now().Before(timeout) { + if _, err := os.Stat(sqlFile); os.IsNotExist(err) { + matchFoundAfterAutoSchemaApply = true + checkTables(t, totalTableCount+1) + matchSchema(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, clusterInstance.Keyspaces[0].Shards[1].Vttablets[0].VttabletProcess.TabletPath) + } + } + if !matchFoundAfterAutoSchemaApply { + assert.Fail(t, "Auto schema is not consumed") + } + defer os.RemoveAll(path.Join(schemaChangeDirectory, keyspaceName)) +} + +// matchSchema schema for supplied tablets should match +func matchSchema(t *testing.T, firstTablet string, secondTablet string) { + firstShardSchema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", firstTablet) + assert.Nil(t, err) + + secondShardSchema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", secondTablet) + assert.Nil(t, err) + + assert.Equal(t, firstShardSchema, secondShardSchema) +} + +// testSchemaChangePreflightErrorPartially applying same schema + new schema should throw error for existing one +// Tests that some SQL statements fail properly during PreflightSchema. +func testSchemaChangePreflightErrorPartially(t *testing.T) { + createNewTable := fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", 5)) + fmt.Sprintf(createTable, fmt.Sprintf("vt_select_test_%02d", 2)) + output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "-sql", createNewTable, keyspaceName) + assert.NotNil(t, err) + assert.True(t, strings.Contains(output, "already exists")) + + checkTables(t, totalTableCount) +} + +// testDropNonExistentTables applying same schema + new schema should throw error for existing one and also add the new schema +//If a table does not exist, DROP TABLE should error during preflight +//because the statement does not change the schema as there is +//nothing to drop. +//In case of DROP TABLE IF EXISTS though, it should not error as this +//is the MySQL behavior the user expects. +func testDropNonExistentTables(t *testing.T) { + dropNonExistentTable := "DROP TABLE nonexistent_table;" + output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ApplySchema", "-sql", dropNonExistentTable, keyspaceName) + assert.NotNil(t, err) + assert.True(t, strings.Contains(output, "Unknown table")) + + dropIfExists := "DROP TABLE IF EXISTS nonexistent_table;" + err = clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, dropIfExists) + assert.Nil(t, err) + + checkTables(t, totalTableCount) +} + +// checkTables checks the number of tables in the first two shards. +func checkTables(t *testing.T, count int) { + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0], count) + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[1].Vttablets[0], count) +} + +// checkTablesCount checks the number of tables in the given tablet +func checkTablesCount(t *testing.T, tablet cluster.Vttablet, count int) { + queryResult, err := tablet.VttabletProcess.QueryTablet("show tables;", keyspaceName, true) + assert.Nil(t, err) + assert.Equal(t, len(queryResult.Rows), count) +} + +// testCopySchemaShards tests that schema from source is correctly applied to destination +func testCopySchemaShards(t *testing.T, source string, shard int) { + addNewShard(t, shard) + // InitShardMaster creates the db, but there shouldn't be any tables yet. + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0], 0) + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[1], 0) + // Run the command twice to make sure it's idempotent. + for i := 0; i < 2; i++ { + err := clusterInstance.VtctlclientProcess.ExecuteCommand("CopySchemaShard", source, fmt.Sprintf("%s/%d", keyspaceName, shard)) + assert.Nil(t, err) + } + // shard_2_master should look the same as the replica we copied from + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0], totalTableCount) + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[1], totalTableCount) + + matchSchema(t, clusterInstance.Keyspaces[0].Shards[0].Vttablets[0].VttabletProcess.TabletPath, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0].VttabletProcess.TabletPath) +} + +// testCopySchemaShardWithDifferentDB if we apply different schema to new shard, it should throw error +func testCopySchemaShardWithDifferentDB(t *testing.T, shard int) { + addNewShard(t, shard) + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0], 0) + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[1], 0) + source := fmt.Sprintf("%s/0", keyspaceName) + + masterTabletAlias := clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0].VttabletProcess.TabletPath + schema, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetSchema", masterTabletAlias) + assert.Nil(t, err) + + resultMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(schema), &resultMap) + assert.Nil(t, err) + dbSchema := reflect.ValueOf(resultMap["database_schema"]) + assert.True(t, strings.Contains(dbSchema.String(), "utf8")) + + // Change the db charset on the destination shard from utf8 to latin1. + // This will make CopySchemaShard fail during its final diff. + // (The different charset won't be corrected on the destination shard + // because we use "CREATE DATABASE IF NOT EXISTS" and this doesn't fail if + // there are differences in the options e.g. the character set.) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ExecuteFetchAsDba", "-json", masterTabletAlias, "ALTER DATABASE vt_ks CHARACTER SET latin1") + assert.Nil(t, err) + + output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("CopySchemaShard", source, fmt.Sprintf("%s/%d", keyspaceName, shard)) + assert.NotNil(t, err) + assert.True(t, strings.Contains(output, "schemas are different")) + + // shard_2_master should have the same number of tables. Only the db + // character set is different. + checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[shard].Vttablets[0], totalTableCount) +} + +// addNewShard adds a new shard dynamically +func addNewShard(t *testing.T, shard int) { + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + err := clusterInstance.StartKeyspace(*keyspace, []string{fmt.Sprintf("%d", shard)}, 1, false) + assert.Nil(t, err) +} diff --git a/go/test/endtoend/vtgate/sequence/seq_test.go b/go/test/endtoend/vtgate/sequence/seq_test.go new file mode 100644 index 00000000000..63eff2f43be --- /dev/null +++ b/go/test/endtoend/vtgate/sequence/seq_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sequence + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + keyspaceName = "ks" + cell = "zone1" + hostname = "localhost" + sqlSchema = ` + create table sequence_test( + id bigint, + val varchar(16), + primary key(id) + )Engine=InnoDB; + + create table sequence_test_seq ( + id int default 0, + next_id bigint default null, + cache bigint default null, + primary key(id) + ) comment 'vitess_sequence' Engine=InnoDB; + ` + + vSchema = ` + { + "sharded":false, + "vindexes": { + "hash_index": { + "type": "hash" + } + }, + "tables": { + "sequence_test":{ + "auto_increment":{ + "column" : "id", + "sequence" : "sequence_test_seq" + }, + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + } + ] + }, + "sequence_test_seq": { + "type": "sequence" + } + } + } + ` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + VSchema: vSchema, + } + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false); err != nil { + return 1 + } + + // Start vtgate + if err := clusterInstance.StartVtgate(); err != nil { + return 1 + } + + return m.Run() + }() + os.Exit(exitCode) +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err) + return qr +} + +func TestSeq(t *testing.T) { + ctx := context.Background() + vtParams := mysql.ConnParams{ + Host: "localhost", + Port: clusterInstance.VtgateMySQLPort, + } + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + //Initialize seq table + exec(t, conn, "insert into sequence_test_seq(id, next_id, cache) values(0,1,10)") + + //Insert 4 values in the main table + exec(t, conn, "insert into sequence_test(val) values('a'), ('b') ,('c'), ('d')") + + // Test select calls to main table and verify expected id. + qr := exec(t, conn, "select id, val from sequence_test where id=4") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(4) VARCHAR("d")]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Test next available seq id from cache + qr = exec(t, conn, "select next 1 values from sequence_test_seq") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(5)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + //Test next_id from seq table which should be the increased by cache value(id+cache) + qr = exec(t, conn, "select next_id from sequence_test_seq") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(11)]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + // Test insert with no auto-inc + exec(t, conn, "insert into sequence_test(id, val) values(6, 'f')") + qr = exec(t, conn, "select * from sequence_test") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(3) VARCHAR("c")] [INT64(4) VARCHAR("d")] [INT64(6) VARCHAR("f")]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } + + //Next insert will fail as we have corrupted the sequence + exec(t, conn, "begin") + _, err = conn.ExecuteFetch("insert into sequence_test(val) values('g')", 1000, false) + exec(t, conn, "rollback") + want := "Duplicate entry" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("wrong insert: %v, must contain %s", err, want) + } + +} diff --git a/go/test/endtoend/vtgate/transaction/trxn_mode_test.go b/go/test/endtoend/vtgate/transaction/trxn_mode_test.go new file mode 100644 index 00000000000..106cb81010c --- /dev/null +++ b/go/test/endtoend/vtgate/transaction/trxn_mode_test.go @@ -0,0 +1,228 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transaction + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceName = "ks" + cell = "zone1" + hostname = "localhost" + sqlSchema = ` + create table twopc_user ( + user_id bigint, + name varchar(128), + primary key (user_id) + ) Engine=InnoDB; + + create table twopc_lookup ( + name varchar(128), + id bigint, + primary key (id) + ) Engine=InnoDB;` + + vSchema = ` + { + "sharded":true, + "vindexes": { + "hash_index": { + "type": "hash" + }, + "twopc_lookup_vdx": { + "type": "lookup_hash_unique", + "params": { + "table": "twopc_lookup", + "from": "name", + "to": "id", + "autocommit": "true" + }, + "owner": "twopc_user" + } + }, + "tables": { + "twopc_user":{ + "column_vindexes": [ + { + "column": "user_id", + "name": "hash_index" + }, + { + "column": "name", + "name": "twopc_lookup_vdx" + } + ] + }, + "twopc_lookup": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + } + ] + } + } + } + ` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitcode, err := func() (int, error) { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Reserve vtGate port in order to pass it to vtTablet + clusterInstance.VtgateGrpcPort = clusterInstance.GetAndReservePort() + // Set extra tablet args for twopc + clusterInstance.VtTabletExtraArgs = []string{ + "-twopc_enable", + "-twopc_coordinator_address", fmt.Sprintf("localhost:%d", clusterInstance.VtgateGrpcPort), + "-twopc_abandon_age", "3600", + } + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return 1, err + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + VSchema: vSchema, + } + if err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false); err != nil { + return 1, err + } + + // Starting Vtgate in SINGLE transaction mode + clusterInstance.VtGateExtraArgs = []string{"-transaction_mode", "SINGLE"} + if err := clusterInstance.StartVtgate(); err != nil { + return 1, err + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + + return m.Run(), nil + }() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } else { + os.Exit(exitcode) + } +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + if err != nil { + t.Fatal(err) + } + return qr +} + +// TestTransactionModes tests trasactions using twopc mode +func TestTransactionModes(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // Insert targeted to multiple tables should fail as Transaction mode is SINGLE + exec(t, conn, "begin") + exec(t, conn, "insert into twopc_user(user_id, name) values(1,'john')") + _, err = conn.ExecuteFetch("insert into twopc_user(user_id, name) values(6,'vick')", 1000, false) + exec(t, conn, "rollback") + want := "multi-db transaction attempted" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("multi-db insert: %v, must contain %s", err, want) + } + + // Enable TWOPC transaction mode + clusterInstance.VtGateExtraArgs = []string{"-transaction_mode", "TWOPC"} + // Restart VtGate + if err = clusterInstance.ReStartVtgate(); err != nil { + t.Errorf("Fail to re-start vtgate with new config: %v", err) + } + + // Make a new mysql connection to vtGate + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + conn2, err := mysql.Connect(ctx, &vtParams) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + + // Insert targeted to multiple db should PASS with TWOPC trx mode + exec(t, conn2, "begin") + exec(t, conn2, "insert into twopc_user(user_id, name) values(3,'mark')") + exec(t, conn2, "insert into twopc_user(user_id, name) values(4,'doug')") + exec(t, conn2, "insert into twopc_lookup(name, id) values('Tim',7)") + exec(t, conn2, "commit") + + // Verify the values are present + qr := exec(t, conn2, "select user_id from twopc_user where name='mark'") + got := fmt.Sprintf("%v", qr.Rows) + want = `[[INT64(3)]]` + assert.Equal(t, want, got) + + qr = exec(t, conn2, "select name from twopc_lookup where id=3") + got = fmt.Sprintf("%v", qr.Rows) + want = `[[VARCHAR("mark")]]` + assert.Equal(t, want, got) + + // DELETE from multiple tables using TWOPC transaction mode + exec(t, conn2, "begin") + exec(t, conn2, "delete from twopc_user where user_id = 3") + exec(t, conn2, "delete from twopc_lookup where id = 3") + exec(t, conn2, "commit") + + // VERIFY that values are deleted + qr = exec(t, conn2, "select user_id from twopc_user where user_id=3") + got = fmt.Sprintf("%v", qr.Rows) + want = `[]` + assert.Equal(t, want, got) + + qr = exec(t, conn2, "select name from twopc_lookup where id=3") + got = fmt.Sprintf("%v", qr.Rows) + want = `[]` + assert.Equal(t, want, got) +} diff --git a/go/test/endtoend/vtgate/vschema/vschema_test.go b/go/test/endtoend/vtgate/vschema/vschema_test.go new file mode 100644 index 00000000000..1938d183e0e --- /dev/null +++ b/go/test/endtoend/vtgate/vschema/vschema_test.go @@ -0,0 +1,169 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vschema + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + hostname = "localhost" + keyspaceName = "ks" + cell = "zone1" + sqlSchema = ` + create table vt_user ( + id bigint, + name varchar(64), + primary key (id) + ) Engine=InnoDB; + + create table main ( + id bigint, + val varchar(128), + primary key(id) + ) Engine=InnoDB; +` +) + +func TestMain(m *testing.M) { + flag.Parse() + + exitcode, err := func() (int, error) { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return 1, err + } + + // List of users authorized to execute vschema ddl operations + clusterInstance.VtGateExtraArgs = []string{"-vschema_ddl_authorized_users=%"} + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + } + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false); err != nil { + return 1, err + } + + // Start vtgate + if err := clusterInstance.StartVtgate(); err != nil { + return 1, err + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run(), nil + }() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } else { + os.Exit(exitcode) + } + +} + +func TestVSchema(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // Test the empty database with no vschema + exec(t, conn, "insert into vt_user (id,name) values(1,'test1'), (2,'test2'), (3,'test3'), (4,'test4')") + + qr := exec(t, conn, "select id, name from vt_user order by id") + got := fmt.Sprintf("%v", qr.Rows) + want := `[[INT64(1) VARCHAR("test1")] [INT64(2) VARCHAR("test2")] [INT64(3) VARCHAR("test3")] [INT64(4) VARCHAR("test4")]]` + assert.Equal(t, want, got) + + qr = exec(t, conn, "delete from vt_user") + got = fmt.Sprintf("%v", qr.Rows) + want = `[]` + assert.Equal(t, want, got) + + // Test empty vschema + qr = exec(t, conn, "SHOW VSCHEMA TABLES") + got = fmt.Sprintf("%v", qr.Rows) + want = `[[VARCHAR("dual")]]` + assert.Equal(t, want, got) + + // Use the DDL to create an unsharded vschema and test again + + // Create VSchema and do a Select to force update VSCHEMA + exec(t, conn, "begin") + exec(t, conn, "ALTER VSCHEMA ADD TABLE vt_user") + exec(t, conn, "select * from vt_user") + exec(t, conn, "commit") + + exec(t, conn, "begin") + exec(t, conn, "ALTER VSCHEMA ADD TABLE main") + exec(t, conn, "select * from main") + exec(t, conn, "commit") + + // Test Showing Tables + qr = exec(t, conn, "SHOW VSCHEMA TABLES") + got = fmt.Sprintf("%v", qr.Rows) + want = `[[VARCHAR("dual")] [VARCHAR("main")] [VARCHAR("vt_user")]]` + assert.Equal(t, want, got) + + // Test Showing Vindexes + qr = exec(t, conn, "SHOW VSCHEMA VINDEXES") + got = fmt.Sprintf("%v", qr.Rows) + want = `[]` + assert.Equal(t, want, got) + + // Test DML operations + exec(t, conn, "insert into vt_user (id,name) values(1,'test1'), (2,'test2'), (3,'test3'), (4,'test4')") + qr = exec(t, conn, "select id, name from vt_user order by id") + got = fmt.Sprintf("%v", qr.Rows) + want = `[[INT64(1) VARCHAR("test1")] [INT64(2) VARCHAR("test2")] [INT64(3) VARCHAR("test3")] [INT64(4) VARCHAR("test4")]]` + assert.Equal(t, want, got) + + qr = exec(t, conn, "delete from vt_user") + got = fmt.Sprintf("%v", qr.Rows) + want = `[]` + assert.Equal(t, want, got) + +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + if err != nil { + t.Fatal(err) + } + return qr +} diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index da92a374289..d7614759a2c 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -111,7 +111,7 @@ func (bps *Stats) MessageHistory() []string { func NewStats() *Stats { bps := &Stats{} bps.Timings = stats.NewTimings("", "", "") - bps.Rates = stats.NewRates("", bps.Timings, 15, 60e9) + bps.Rates = stats.NewRates("", bps.Timings, 15*60/5, 5*time.Second) bps.History = history.New(3) bps.SecondsBehindMaster.Set(math.MaxInt64) return bps @@ -202,6 +202,7 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { log.Error(err) return err } + blp.position = settings.StartPos blp.stopPosition = settings.StopPos t, err := throttler.NewThrottler( @@ -508,7 +509,7 @@ func AlterVReplicationTable() []string { // SetVReplicationState updates the state in the _vt.vreplication table. func SetVReplicationState(dbClient DBClient, uid uint32, state, message string) error { - query := fmt.Sprintf("update _vt.vreplication set state='%v', message=%v where id=%v", state, encodeString(message), uid) + query := fmt.Sprintf("update _vt.vreplication set state='%v', message=%v where id=%v", state, encodeString(MessageTruncate(message)), uid) if _, err := dbClient.ExecuteFetch(query, 1); err != nil { return fmt.Errorf("could not set state: %v: %v", query, err) } @@ -613,7 +614,7 @@ func StartVReplicationUntil(uid uint32, pos string) string { func StopVReplication(uid uint32, message string) string { return fmt.Sprintf( "update _vt.vreplication set state='%v', message=%v where id=%v", - BlpStopped, encodeString(message), uid) + BlpStopped, encodeString(MessageTruncate(message)), uid) } // DeleteVReplication returns a statement to delete the replication. @@ -621,6 +622,15 @@ func DeleteVReplication(uid uint32) string { return fmt.Sprintf("delete from _vt.vreplication where id=%v", uid) } +// MessageTruncate truncates the message string to a safe length. +func MessageTruncate(msg string) string { + // message length is 1000 bytes. + if len(msg) > 950 { + return msg[:950] + "..." + } + return msg +} + func encodeString(in string) string { buf := bytes.NewBuffer(nil) sqltypes.NewVarChar(in).EncodeSQL(buf) diff --git a/go/vt/binlog/keyspace_id_resolver.go b/go/vt/binlog/keyspace_id_resolver.go index 204960f06ca..a005222434d 100644 --- a/go/vt/binlog/keyspace_id_resolver.go +++ b/go/vt/binlog/keyspace_id_resolver.go @@ -147,7 +147,8 @@ func newKeyspaceIDResolverFactoryV3(ctx context.Context, ts *topo.Server, keyspa if col.Name.EqualString(shardingColumnName) { // We found the column. return i, &keyspaceIDResolverFactoryV3{ - vindex: colVindex.Vindex, + // Only SingleColumn vindexes are returned by FindVindexForSharding. + vindex: colVindex.Vindex.(vindexes.SingleColumn), }, nil } } @@ -158,7 +159,7 @@ func newKeyspaceIDResolverFactoryV3(ctx context.Context, ts *topo.Server, keyspa // keyspaceIDResolverFactoryV3 uses the Vindex to compute the value. type keyspaceIDResolverFactoryV3 struct { - vindex vindexes.Vindex + vindex vindexes.SingleColumn } func (r *keyspaceIDResolverFactoryV3) keyspaceID(v sqltypes.Value) ([]byte, error) { diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index 2976f05c30d..c9df1ab26c8 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -69,14 +69,15 @@ const ( // AllPrivs user should have more privileges than App (should include possibility to do // schema changes and write to internal Vitess tables), but it shouldn't have SUPER // privilege like Dba has. - AllPrivs = "allprivs" - Dba = "dba" - Filtered = "filtered" - Repl = "repl" + AllPrivs = "allprivs" + Dba = "dba" + Filtered = "filtered" + Repl = "repl" + ExternalRepl = "erepl" ) // All can be used to register all flags: RegisterFlags(All...) -var All = []string{App, AppDebug, AllPrivs, Dba, Filtered, Repl} +var All = []string{App, AppDebug, AllPrivs, Dba, Filtered, Repl, ExternalRepl} // RegisterFlags registers the flags for the given DBConfigFlag. // For instance, vttablet will register client, dba and repl. @@ -96,6 +97,7 @@ func registerBaseFlags() { flag.IntVar(&baseConfig.Port, "db_port", 0, "tcp port") flag.StringVar(&baseConfig.Charset, "db_charset", "", "Character set. Only utf8 or latin1 based character sets are supported.") flag.Uint64Var(&baseConfig.Flags, "db_flags", 0, "Flag values as defined by MySQL.") + flag.StringVar(&baseConfig.Flavor, "db_flavor", "", "Flavor overrid. Valid value is FilePos.") flag.StringVar(&baseConfig.SslCa, "db_ssl_ca", "", "connection ssl ca") flag.StringVar(&baseConfig.SslCaPath, "db_ssl_ca_path", "", "connection ssl ca path") flag.StringVar(&baseConfig.SslCert, "db_ssl_cert", "", "connection ssl certificate") @@ -127,6 +129,7 @@ func registerPerUserFlags(dbc *userConfig, userKey string) { flag.StringVar(&dbc.param.SslCert, "db-config-"+userKey+"-ssl-cert", "", "deprecated: use db_ssl_cert") flag.StringVar(&dbc.param.SslKey, "db-config-"+userKey+"-ssl-key", "", "deprecated: use db_ssl_key") flag.StringVar(&dbc.param.ServerName, "db-config-"+userKey+"-server_name", "", "deprecated: use db_server_name") + flag.StringVar(&dbc.param.Flavor, "db-config-"+userKey+"-flavor", "", "deprecated: use db_flavor") flag.StringVar(&dbc.param.DeprecatedDBName, "db-config-"+userKey+"-dbname", "", "deprecated: dbname does not need to be explicitly configured") @@ -157,16 +160,33 @@ func (dbcfgs *DBConfigs) DbaWithDB() *mysql.ConnParams { return dbcfgs.makeParams(Dba, true) } -// FilteredWithDB returns connection parameters for appdebug with dbname set. +// FilteredWithDB returns connection parameters for filtered with dbname set. func (dbcfgs *DBConfigs) FilteredWithDB() *mysql.ConnParams { return dbcfgs.makeParams(Filtered, true) } -// Repl returns connection parameters for appdebug with no dbname set. +// Repl returns connection parameters for repl with no dbname set. func (dbcfgs *DBConfigs) Repl() *mysql.ConnParams { return dbcfgs.makeParams(Repl, false) } +// ExternalRepl returns connection parameters for repl with no dbname set. +func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { + return dbcfgs.makeParams(ExternalRepl, true) +} + +// ExternalReplWithDB returns connection parameters for repl with dbname set. +func (dbcfgs *DBConfigs) ExternalReplWithDB() *mysql.ConnParams { + params := dbcfgs.makeParams(ExternalRepl, true) + // TODO @rafael: This is a hack to allows to configure external databases by providing + // db-config-erepl-dbname. + if params.DeprecatedDBName != "" { + params.DbName = params.DeprecatedDBName + return params + } + return params +} + // AppWithDB returns connection parameters for app with dbname set. func (dbcfgs *DBConfigs) makeParams(userKey string, withDB bool) *mysql.ConnParams { orig := dbcfgs.userConfigs[userKey] @@ -237,8 +257,13 @@ func HasConnectionParams() bool { // is used to initialize the per-user conn params. func Init(defaultSocketFile string) (*DBConfigs, error) { // The new base configs, if set, supersede legacy settings. - for _, uc := range dbConfigs.userConfigs { - if HasConnectionParams() { + for user, uc := range dbConfigs.userConfigs { + // TODO @rafael: For ExternalRepl we need to respect the provided host / port + // At the moment this is an snowflake user connection type that it used by + // vreplication to connect to external mysql hosts that are not part of a vitess + // cluster. In the future we need to refactor all dbconfig to support custom users + // in a more flexible way. + if HasConnectionParams() && user != ExternalRepl { uc.param.Host = baseConfig.Host uc.param.Port = baseConfig.Port uc.param.UnixSocket = baseConfig.UnixSocket @@ -252,6 +277,9 @@ func Init(defaultSocketFile string) (*DBConfigs, error) { if baseConfig.Flags != 0 { uc.param.Flags = baseConfig.Flags } + if user != ExternalRepl { + uc.param.Flavor = baseConfig.Flavor + } if uc.useSSL { uc.param.SslCa = baseConfig.SslCa uc.param.SslCaPath = baseConfig.SslCaPath @@ -280,12 +308,13 @@ func Init(defaultSocketFile string) (*DBConfigs, error) { func NewTestDBConfigs(genParams, appDebugParams mysql.ConnParams, dbName string) *DBConfigs { dbcfgs := &DBConfigs{ userConfigs: map[string]*userConfig{ - App: {param: genParams}, - AppDebug: {param: appDebugParams}, - AllPrivs: {param: genParams}, - Dba: {param: genParams}, - Filtered: {param: genParams}, - Repl: {param: genParams}, + App: {param: genParams}, + AppDebug: {param: appDebugParams}, + AllPrivs: {param: genParams}, + Dba: {param: genParams}, + Filtered: {param: genParams}, + Repl: {param: genParams}, + ExternalRepl: {param: genParams}, }, } dbcfgs.DBName.Set(dbName) diff --git a/go/vt/dbconfigs/dbconfigs_test.go b/go/vt/dbconfigs/dbconfigs_test.go index 54b413b9038..d20b50539f8 100644 --- a/go/vt/dbconfigs/dbconfigs_test.go +++ b/go/vt/dbconfigs/dbconfigs_test.go @@ -75,6 +75,7 @@ func TestInit(t *testing.T) { UnixSocket: "e", Charset: "f", Flags: 2, + Flavor: "flavor", SslCa: "g", SslCaPath: "h", SslCert: "i", @@ -117,6 +118,7 @@ func TestInit(t *testing.T) { UnixSocket: "e", Charset: "f", Flags: 2, + Flavor: "flavor", }, }, AppDebug: { @@ -127,6 +129,7 @@ func TestInit(t *testing.T) { UnixSocket: "e", Charset: "f", Flags: 2, + Flavor: "flavor", SslCa: "g", SslCaPath: "h", SslCert: "i", @@ -143,6 +146,7 @@ func TestInit(t *testing.T) { UnixSocket: "e", Charset: "f", Flags: 2, + Flavor: "flavor", SslCa: "g", SslCaPath: "h", SslCert: "i", diff --git a/go/vt/env/env.go b/go/vt/env/env.go index 49b462815be..dd28d1eb5c1 100644 --- a/go/vt/env/env.go +++ b/go/vt/env/env.go @@ -18,7 +18,9 @@ package env import ( "errors" + "fmt" "os" + "os/exec" "path" "path/filepath" "strings" @@ -60,20 +62,26 @@ func VtDataRoot() string { return DefaultVtDataRoot } -// VtMysqlRoot returns the root for the mysql distribution, which -// contains bin/mysql CLI for instance. +// VtMysqlRoot returns the root for the mysql distribution, +// which contains bin/mysql CLI for instance. +// If it is not set, look for mysqld in the path. func VtMysqlRoot() (string, error) { // if the environment variable is set, use that if root := os.Getenv("VT_MYSQL_ROOT"); root != "" { return root, nil } - // otherwise let's use VTROOT - root, err := VtRoot() + // otherwise let's look for mysqld in the PATH. + // ensure that /usr/sbin is included, as it might not be by default + // This is the default location for mysqld from packages. + newPath := fmt.Sprintf("/usr/sbin:%s", os.Getenv("PATH")) + os.Setenv("PATH", newPath) + path, err := exec.LookPath("mysqld") if err != nil { - return "", errors.New("VT_MYSQL_ROOT is not set and could not be guessed from the executable location. Please set $VT_MYSQL_ROOT") + return "", errors.New("VT_MYSQL_ROOT is not set and no mysqld could be found in your PATH") } - return root, nil + path = filepath.Dir(filepath.Dir(path)) // strip mysqld, and the sbin + return path, nil } // VtMysqlBaseDir returns the Mysql base directory, which diff --git a/go/vt/key/key.go b/go/vt/key/key.go index b9ae9698403..0bd7f137a34 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -120,6 +120,21 @@ func EvenShardsKeyRange(i, n int) (*topodatapb.KeyRange, error) { return &topodatapb.KeyRange{Start: startBytes, End: endBytes}, nil } +// KeyRangeAdd adds two adjacent keyranges into a single value. +// If the values are not adjacent, it returns false. +func KeyRangeAdd(first, second *topodatapb.KeyRange) (*topodatapb.KeyRange, bool) { + if first == nil || second == nil { + return nil, false + } + if len(first.End) != 0 && bytes.Equal(first.End, second.Start) { + return &topodatapb.KeyRange{Start: first.Start, End: second.End}, true + } + if len(second.End) != 0 && bytes.Equal(second.End, first.Start) { + return &topodatapb.KeyRange{Start: second.Start, End: first.End}, true + } + return nil, false +} + // KeyRangeContains returns true if the provided id is in the keyrange. func KeyRangeContains(kr *topodatapb.KeyRange, id []byte) bool { if kr == nil { diff --git a/go/vt/key/key_test.go b/go/vt/key/key_test.go index 9d88ba19ea3..89bac6311fc 100644 --- a/go/vt/key/key_test.go +++ b/go/vt/key/key_test.go @@ -139,6 +139,107 @@ func TestEvenShardsKeyRange(t *testing.T) { } } +func TestKeyRangeAdd(t *testing.T) { + testcases := []struct { + first string + second string + out string + ok bool + }{{ + first: "", + second: "", + out: "", + ok: false, + }, { + first: "", + second: "-80", + out: "", + ok: false, + }, { + first: "-80", + second: "", + out: "", + ok: false, + }, { + first: "", + second: "80-", + out: "", + ok: false, + }, { + first: "80-", + second: "", + out: "", + ok: false, + }, { + first: "80-", + second: "-40", + out: "", + ok: false, + }, { + first: "-40", + second: "80-", + out: "", + ok: false, + }, { + first: "-80", + second: "80-", + out: "-", + ok: true, + }, { + first: "80-", + second: "-80", + out: "-", + ok: true, + }, { + first: "-40", + second: "40-80", + out: "-80", + ok: true, + }, { + first: "40-80", + second: "-40", + out: "-80", + ok: true, + }, { + first: "40-80", + second: "80-c0", + out: "40-c0", + ok: true, + }, { + first: "80-c0", + second: "40-80", + out: "40-c0", + ok: true, + }} + stringToKeyRange := func(spec string) *topodatapb.KeyRange { + if spec == "" { + return nil + } + parts := strings.Split(spec, "-") + if len(parts) != 2 { + panic("invalid spec") + } + kr, err := ParseKeyRangeParts(parts[0], parts[1]) + if err != nil { + panic(err) + } + return kr + } + keyRangeToString := func(kr *topodatapb.KeyRange) string { + if kr == nil { + return "" + } + return KeyRangeString(kr) + } + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out, ok := KeyRangeAdd(first, second) + assert.Equal(t, tcase.out, keyRangeToString(out)) + assert.Equal(t, tcase.ok, ok) + } +} + func TestEvenShardsKeyRange_Error(t *testing.T) { testCases := []struct { i, n int diff --git a/go/vt/mysqlctl/backupstorage/interface.go b/go/vt/mysqlctl/backupstorage/interface.go index 03e7b64949c..e4c2e6bc18d 100644 --- a/go/vt/mysqlctl/backupstorage/interface.go +++ b/go/vt/mysqlctl/backupstorage/interface.go @@ -30,6 +30,10 @@ var ( // BackupStorageImplementation is the implementation to use // for BackupStorage. Exported for test purposes. BackupStorageImplementation = flag.String("backup_storage_implementation", "", "which implementation to use for the backup storage feature") + // FileSizeUnknown is a special value indicating that the file size is not known. + // This is typically used while creating a file programmatically, where it is + // impossible to compute the final size on disk ahead of time. + FileSizeUnknown = int64(-1) ) // BackupHandle describes an individual backup. @@ -50,7 +54,9 @@ type BackupHandle interface { // The context is valid for the duration of the writes, until the // WriteCloser is closed. // filesize should not be treated as an exact value but rather - // as an approximate value + // as an approximate value. + // A filesize of -1 should be treated as a special value indicating that + // the file size is unknown. AddFile(ctx context.Context, filename string, filesize int64) (io.WriteCloser, error) // EndBackup stops and closes a backup. The contents should be kept. diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 5f8af216dc0..f49f8fdf780 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -303,7 +303,7 @@ func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, params BackupPar } // open the MANIFEST - wc, err := bh.AddFile(ctx, backupManifestFileName, 0) + wc, err := bh.AddFile(ctx, backupManifestFileName, backupstorage.FileSizeUnknown) if err != nil { return vterrors.Wrapf(err, "cannot add %v to backup", backupManifestFileName) } diff --git a/go/vt/mysqlctl/capabilityset.go b/go/vt/mysqlctl/capabilityset.go index 909eec70ac3..1b0855e3c1c 100644 --- a/go/vt/mysqlctl/capabilityset.go +++ b/go/vt/mysqlctl/capabilityset.go @@ -46,6 +46,9 @@ func (c *capabilitySet) hasMySQLUpgradeInServer() bool { func (c *capabilitySet) hasInitializeInServer() bool { return c.isMySQLLike() && c.version.atLeast(serverVersion{Major: 5, Minor: 7, Patch: 0}) } +func (c *capabilitySet) hasMaria104InstallDb() bool { + return c.isMariaDB() && c.version.atLeast(serverVersion{Major: 10, Minor: 4, Patch: 0}) +} // IsMySQLLike tests if the server is either MySQL // or Percona Server. At least currently, Vitess doesn't @@ -53,3 +56,6 @@ func (c *capabilitySet) hasInitializeInServer() bool { func (c *capabilitySet) isMySQLLike() bool { return c.flavor == flavorMySQL || c.flavor == flavorPercona } +func (c *capabilitySet) isMariaDB() bool { + return c.flavor == flavorMariaDB +} diff --git a/go/vt/mysqlctl/cephbackupstorage/ceph.go b/go/vt/mysqlctl/cephbackupstorage/ceph.go index 29b76e1b15e..80c37ceb828 100644 --- a/go/vt/mysqlctl/cephbackupstorage/ceph.go +++ b/go/vt/mysqlctl/cephbackupstorage/ceph.go @@ -88,7 +88,8 @@ func (bh *CephBackupHandle) AddFile(ctx context.Context, filename string, filesi // Give PutObject() the read end of the pipe. object := objName(bh.dir, bh.name, filename) - _, err := bh.client.PutObjectWithContext(ctx, bucket, object, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + // If filesize is unknown, the caller should pass in -1 and we will pass it through. + _, err := bh.client.PutObjectWithContext(ctx, bucket, object, reader, filesize, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { // Signal the writer that an error occurred, in case it's not done writing yet. reader.CloseWithError(err) diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go index e7215f894fe..59c4247eacb 100644 --- a/go/vt/mysqlctl/mycnf_test.go +++ b/go/vt/mysqlctl/mycnf_test.go @@ -42,9 +42,7 @@ func TestMycnf(t *testing.T) { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ - path.Join(root, "src/vitess.io/vitess/config/mycnf/default.cnf"), - path.Join(root, "src/vitess.io/vitess/config/mycnf/replica.cnf"), - path.Join(root, "src/vitess.io/vitess/config/mycnf/master.cnf"), + path.Join(root, "config/mycnf/default.cnf"), } data, err := cnf.makeMycnf(cnfTemplatePaths) if err != nil { @@ -81,7 +79,7 @@ func TestMycnf(t *testing.T) { // Run this test if any changes are made to hook handling / make_mycnf hook // other tests fail if we keep the hook around -// 1. ln -snf $VTTOP/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf +// 1. ln -snf $VTROOT/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf // 2. Remove "No" prefix from func name // 3. go test // 4. \rm $VTROOT/vthook/make_mycnf diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 3402829441d..4f18809c04b 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -133,8 +133,8 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { /* By default Vitess searches in vtenv.VtMysqlRoot() for a mysqld binary. - This is usually the VT_MYSQL_ROOT env, but if it is unset or empty, it - will substitute VtRoot(). See go/vt/env/env.go. + This is historically the VT_MYSQL_ROOT env, but if it is unset or empty, + Vitess will search the PATH. See go/vt/env/env.go. A number of subdirs inside vtenv.VtMysqlRoot() will be searched, see func binaryPath() for context. If no mysqld binary is found (possibly @@ -153,12 +153,14 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { f, v, err = getVersionFromEnv() if err != nil { vtenvMysqlRoot, _ := vtenv.VtMysqlRoot() - message := fmt.Sprintf(`could not auto-detect MySQL version. You may need to set VT_MYSQL_ROOT so a mysqld binary can be found, or set the environment variable MYSQL_FLAVOR if mysqld is not available locally: + message := fmt.Sprintf(`could not auto-detect MySQL version. You may need to set your PATH so a mysqld binary can be found, or set the environment variable MYSQL_FLAVOR if mysqld is not available locally: + PATH: %s VT_MYSQL_ROOT: %s VTROOT: %s vtenv.VtMysqlRoot(): %s MYSQL_FLAVOR: %s `, + os.Getenv("PATH"), os.Getenv("VT_MYSQL_ROOT"), os.Getenv("VTROOT"), vtenvMysqlRoot, @@ -265,12 +267,12 @@ func (mysqld *Mysqld) RunMysqlUpgrade() error { } // Find mysql_upgrade. If not there, we do nothing. - dir, err := vtenv.VtMysqlRoot() + vtMysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { log.Warningf("VT_MYSQL_ROOT not set, skipping mysql_upgrade step: %v", err) return nil } - name, err := binaryPath(dir, "mysql_upgrade") + name, err := binaryPath(vtMysqlRoot, "mysql_upgrade") if err != nil { log.Warningf("mysql_upgrade binary not present, skipping it: %v", err) return nil @@ -301,7 +303,8 @@ func (mysqld *Mysqld) RunMysqlUpgrade() error { "--force", // Don't complain if it's already been upgraded. } cmd := exec.Command(name, args...) - cmd.Env = []string{os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql")} + libPath := fmt.Sprintf("LD_LIBRARY_PATH=%s/lib/mysql", vtMysqlRoot) + cmd.Env = []string{libPath} out, err := cmd.CombinedOutput() log.Infof("mysql_upgrade output: %s", out) return err @@ -344,16 +347,16 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. case hook.HOOK_DOES_NOT_EXIST: // hook doesn't exist, run mysqld_safe ourselves log.Infof("%v: No mysqld_start hook, running mysqld_safe directly", ts) - dir, err := vtenv.VtMysqlRoot() + vtMysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { return err } - name, err = binaryPath(dir, "mysqld_safe") + name, err = binaryPath(vtMysqlRoot, "mysqld_safe") if err != nil { // The movement to use systemd means that mysqld_safe is not always provided. // This should not be considered an issue do not generate a warning. log.Infof("%v: trying to launch mysqld instead", err) - name, err = binaryPath(dir, "mysqld") + name, err = binaryPath(vtMysqlRoot, "mysqld") // If this also fails, return an error. if err != nil { return err @@ -368,10 +371,11 @@ func (mysqld *Mysqld) startNoWait(ctx context.Context, cnf *Mycnf, mysqldArgs .. "--basedir=" + mysqlBaseDir, } arg = append(arg, mysqldArgs...) - env := []string{os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql")} + libPath := fmt.Sprintf("LD_LIBRARY_PATH=%s/lib/mysql", vtMysqlRoot) + env := []string{libPath} cmd := exec.Command(name, arg...) - cmd.Dir = dir + cmd.Dir = vtMysqlRoot cmd.Env = env log.Infof("%v %#v", ts, cmd) stderr, err := cmd.StderrPipe() @@ -742,6 +746,9 @@ func (mysqld *Mysqld) installDataDir(cnf *Mycnf) error { "--defaults-file=" + cnf.path, "--basedir=" + mysqlBaseDir, } + if mysqld.capabilities.hasMaria104InstallDb() { + args = append(args, "--auth-root-authentication-method=normal") + } cmdPath, err := binaryPath(mysqlRoot, "mysql_install_db") if err != nil { return err @@ -795,8 +802,6 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string { cnfTemplatePaths := []string{ path.Join(root, "config/mycnf/default.cnf"), - path.Join(root, "config/mycnf/master.cnf"), - path.Join(root, "config/mycnf/replica.cnf"), } if extraCnf := os.Getenv("EXTRA_MY_CNF"); extraCnf != "" { diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index fc9e27e2523..08d18638bce 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -158,7 +158,7 @@ func (be *XtrabackupEngine) ExecuteBackup(ctx context.Context, params BackupPara // open the MANIFEST params.Logger.Infof("Writing backup MANIFEST") - mwc, err := bh.AddFile(ctx, backupManifestFileName, 0) + mwc, err := bh.AddFile(ctx, backupManifestFileName, backupstorage.FileSizeUnknown) if err != nil { return false, vterrors.Wrapf(err, "cannot add %v to backup", backupManifestFileName) } diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 904d001d423..22b36bc0c54 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -716,10 +716,13 @@ type BinlogSource struct { // for the filter. Filter *Filter `protobuf:"bytes,6,opt,name=filter,proto3" json:"filter,omitempty"` // on_ddl specifies the action to be taken when a DDL is encountered. - OnDdl OnDDLAction `protobuf:"varint,7,opt,name=on_ddl,json=onDdl,proto3,enum=binlogdata.OnDDLAction" json:"on_ddl,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + OnDdl OnDDLAction `protobuf:"varint,7,opt,name=on_ddl,json=onDdl,proto3,enum=binlogdata.OnDDLAction" json:"on_ddl,omitempty"` + // Source is an external mysql. This attribute should be set to the username + // to use in the connection + ExternalMysql string `protobuf:"bytes,8,opt,name=external_mysql,json=externalMysql,proto3" json:"external_mysql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *BinlogSource) Reset() { *m = BinlogSource{} } @@ -796,6 +799,13 @@ func (m *BinlogSource) GetOnDdl() OnDDLAction { return OnDDLAction_IGNORE } +func (m *BinlogSource) GetExternalMysql() string { + if m != nil { + return m.ExternalMysql + } + return "" +} + // RowChange represents one row change type RowChange struct { Before *query.Row `protobuf:"bytes,1,opt,name=before,proto3" json:"before,omitempty"` @@ -1177,6 +1187,7 @@ type VEvent struct { FieldEvent *FieldEvent `protobuf:"bytes,6,opt,name=field_event,json=fieldEvent,proto3" json:"field_event,omitempty"` Vgtid *VGtid `protobuf:"bytes,7,opt,name=vgtid,proto3" json:"vgtid,omitempty"` Journal *Journal `protobuf:"bytes,8,opt,name=journal,proto3" json:"journal,omitempty"` + Dml string `protobuf:"bytes,9,opt,name=dml,proto3" json:"dml,omitempty"` // current_time specifies the current time to handle clock skew. CurrentTime int64 `protobuf:"varint,20,opt,name=current_time,json=currentTime,proto3" json:"current_time,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1265,6 +1276,13 @@ func (m *VEvent) GetJournal() *Journal { return nil } +func (m *VEvent) GetDml() string { + if m != nil { + return m.Dml + } + return "" +} + func (m *VEvent) GetCurrentTime() int64 { if m != nil { return m.CurrentTime @@ -1685,108 +1703,110 @@ func init() { func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_5fd02bcb2e350dad) } var fileDescriptor_5fd02bcb2e350dad = []byte{ - // 1648 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0x4b, 0x73, 0xe3, 0x4a, - 0x15, 0x8e, 0x6c, 0xf9, 0x75, 0x94, 0x38, 0x4a, 0xe7, 0x81, 0x49, 0x71, 0xa9, 0x5c, 0x15, 0x97, - 0xe4, 0xa6, 0x0a, 0x07, 0x0c, 0x0c, 0xab, 0xcb, 0xc5, 0x0f, 0x25, 0x71, 0x22, 0xdb, 0x99, 0xb6, - 0x92, 0xa1, 0x66, 0xa3, 0x52, 0xec, 0x76, 0x22, 0x22, 0x4b, 0x1e, 0xa9, 0x9d, 0x90, 0x1f, 0x40, - 0xf1, 0x03, 0xd8, 0xf2, 0x07, 0x58, 0xb3, 0x85, 0x2d, 0x7b, 0xf6, 0x54, 0xb1, 0xe2, 0x7f, 0x50, - 0xfd, 0x90, 0x6c, 0x25, 0xc3, 0x4c, 0x66, 0xaa, 0x58, 0xc0, 0xc6, 0x75, 0xfa, 0xf4, 0x39, 0xa7, - 0xcf, 0xf9, 0xce, 0xa3, 0xd5, 0x06, 0xfd, 0xda, 0x0b, 0xfc, 0xf0, 0x66, 0xec, 0x52, 0xb7, 0x3e, - 0x8b, 0x42, 0x1a, 0x22, 0x58, 0x70, 0x76, 0xb5, 0x7b, 0x1a, 0xcd, 0x46, 0x62, 0x63, 0x57, 0x7b, - 0x37, 0x27, 0xd1, 0xa3, 0x5c, 0x54, 0x69, 0x38, 0x0b, 0x17, 0x5a, 0x46, 0x0f, 0x4a, 0xed, 0x5b, - 0x37, 0x8a, 0x09, 0x45, 0x3b, 0x50, 0x1c, 0xf9, 0x1e, 0x09, 0x68, 0x4d, 0xd9, 0x53, 0x0e, 0x0a, - 0x58, 0xae, 0x10, 0x02, 0x75, 0x14, 0x06, 0x41, 0x2d, 0xc7, 0xb9, 0x9c, 0x66, 0xb2, 0x31, 0x89, - 0xee, 0x49, 0x54, 0xcb, 0x0b, 0x59, 0xb1, 0x32, 0xfe, 0x95, 0x87, 0x8d, 0x16, 0xf7, 0xc3, 0x8e, - 0xdc, 0x20, 0x76, 0x47, 0xd4, 0x0b, 0x03, 0x74, 0x02, 0x10, 0x53, 0x97, 0x92, 0x29, 0x09, 0x68, - 0x5c, 0x53, 0xf6, 0xf2, 0x07, 0x5a, 0x63, 0xbf, 0xbe, 0x14, 0xc1, 0x33, 0x95, 0xfa, 0x30, 0x91, - 0xc7, 0x4b, 0xaa, 0xa8, 0x01, 0x1a, 0xb9, 0x27, 0x01, 0x75, 0x68, 0x78, 0x47, 0x82, 0x9a, 0xba, - 0xa7, 0x1c, 0x68, 0x8d, 0x8d, 0xba, 0x08, 0xd0, 0x64, 0x3b, 0x36, 0xdb, 0xc0, 0x40, 0x52, 0x7a, - 0xf7, 0x6f, 0x39, 0xa8, 0xa4, 0xd6, 0x90, 0x05, 0xe5, 0x91, 0x4b, 0xc9, 0x4d, 0x18, 0x3d, 0xf2, - 0x30, 0xab, 0x8d, 0x1f, 0xbf, 0xd0, 0x91, 0x7a, 0x5b, 0xea, 0xe1, 0xd4, 0x02, 0xfa, 0x11, 0x94, - 0x46, 0x02, 0x3d, 0x8e, 0x8e, 0xd6, 0xd8, 0x5c, 0x36, 0x26, 0x81, 0xc5, 0x89, 0x0c, 0xd2, 0x21, - 0x1f, 0xbf, 0xf3, 0x39, 0x64, 0xab, 0x98, 0x91, 0xc6, 0x9f, 0x14, 0x28, 0x27, 0x76, 0xd1, 0x26, - 0xac, 0xb7, 0x2c, 0xe7, 0xb2, 0x8f, 0xcd, 0xf6, 0xe0, 0xa4, 0xdf, 0x7d, 0x6b, 0x76, 0xf4, 0x15, - 0xb4, 0x0a, 0xe5, 0x96, 0xe5, 0xb4, 0xcc, 0x93, 0x6e, 0x5f, 0x57, 0xd0, 0x1a, 0x54, 0x5a, 0x96, - 0xd3, 0x1e, 0xf4, 0x7a, 0x5d, 0x5b, 0xcf, 0xa1, 0x75, 0xd0, 0x5a, 0x96, 0x83, 0x07, 0x96, 0xd5, - 0x6a, 0xb6, 0xcf, 0xf5, 0x3c, 0xda, 0x86, 0x8d, 0x96, 0xe5, 0x74, 0x7a, 0x96, 0xd3, 0x31, 0x2f, - 0xb0, 0xd9, 0x6e, 0xda, 0x66, 0x47, 0x57, 0x11, 0x40, 0x91, 0xb1, 0x3b, 0x96, 0x5e, 0x90, 0xf4, - 0xd0, 0xb4, 0xf5, 0xa2, 0x34, 0xd7, 0xed, 0x0f, 0x4d, 0x6c, 0xeb, 0x25, 0xb9, 0xbc, 0xbc, 0xe8, - 0x34, 0x6d, 0x53, 0x2f, 0xcb, 0x65, 0xc7, 0xb4, 0x4c, 0xdb, 0xd4, 0x2b, 0x67, 0x6a, 0x39, 0xa7, - 0xe7, 0xcf, 0xd4, 0x72, 0x5e, 0x57, 0x8d, 0x3f, 0x28, 0xb0, 0x3d, 0xa4, 0x11, 0x71, 0xa7, 0xe7, - 0xe4, 0x11, 0xbb, 0xc1, 0x0d, 0xc1, 0xe4, 0xdd, 0x9c, 0xc4, 0x14, 0xed, 0x42, 0x79, 0x16, 0xc6, - 0x1e, 0xc3, 0x8e, 0x03, 0x5c, 0xc1, 0xe9, 0x1a, 0x1d, 0x41, 0xe5, 0x8e, 0x3c, 0x3a, 0x11, 0x93, - 0x97, 0x80, 0xa1, 0x7a, 0x5a, 0x90, 0xa9, 0xa5, 0xf2, 0x9d, 0xa4, 0x96, 0xf1, 0xcd, 0x7f, 0x1c, - 0x5f, 0x63, 0x02, 0x3b, 0x4f, 0x9d, 0x8a, 0x67, 0x61, 0x10, 0x13, 0x64, 0x01, 0x12, 0x8a, 0x0e, - 0x5d, 0xe4, 0x96, 0xfb, 0xa7, 0x35, 0xbe, 0xf8, 0x60, 0x01, 0xe0, 0x8d, 0xeb, 0xa7, 0x2c, 0xe3, - 0xb7, 0xb0, 0x29, 0xce, 0xb1, 0xdd, 0x6b, 0x9f, 0xc4, 0x2f, 0x09, 0x7d, 0x07, 0x8a, 0x94, 0x0b, - 0xd7, 0x72, 0x7b, 0xf9, 0x83, 0x0a, 0x96, 0xab, 0x4f, 0x8d, 0x70, 0x0c, 0x5b, 0xd9, 0x93, 0xff, - 0x2b, 0xf1, 0xfd, 0x0c, 0x54, 0x3c, 0xf7, 0x09, 0xda, 0x82, 0xc2, 0xd4, 0xa5, 0xa3, 0x5b, 0x19, - 0x8d, 0x58, 0xb0, 0x50, 0x26, 0x9e, 0x4f, 0x49, 0xc4, 0x53, 0x58, 0xc1, 0x72, 0x65, 0xfc, 0x59, - 0x81, 0xe2, 0x31, 0x27, 0xd1, 0x0f, 0xa1, 0x10, 0xcd, 0x59, 0xb0, 0xa2, 0xd7, 0xf5, 0x65, 0x0f, - 0x98, 0x65, 0x2c, 0xb6, 0x51, 0x17, 0xaa, 0x13, 0x8f, 0xf8, 0x63, 0xde, 0xba, 0xbd, 0x70, 0x2c, - 0xaa, 0xa2, 0xda, 0xf8, 0x72, 0x59, 0x41, 0xd8, 0xac, 0x1f, 0x67, 0x04, 0xf1, 0x13, 0x45, 0xe3, - 0x15, 0x54, 0xb3, 0x12, 0xac, 0x9d, 0x4c, 0x8c, 0x9d, 0x41, 0xdf, 0xe9, 0x75, 0x87, 0xbd, 0xa6, - 0xdd, 0x3e, 0xd5, 0x57, 0x78, 0xc7, 0x98, 0x43, 0xdb, 0x31, 0x8f, 0x8f, 0x07, 0xd8, 0xd6, 0x15, - 0xe3, 0x8f, 0x39, 0x58, 0x15, 0xa0, 0x0c, 0xc3, 0x79, 0x34, 0x22, 0x2c, 0x8b, 0x77, 0xe4, 0x31, - 0x9e, 0xb9, 0x23, 0x92, 0x64, 0x31, 0x59, 0x33, 0x40, 0xe2, 0x5b, 0x37, 0x1a, 0xcb, 0xc8, 0xc5, - 0x02, 0xfd, 0x1c, 0x34, 0x9e, 0x4d, 0xea, 0xd0, 0xc7, 0x19, 0xe1, 0x79, 0xac, 0x36, 0xb6, 0x16, - 0x85, 0xcd, 0x73, 0x45, 0xed, 0xc7, 0x19, 0xc1, 0x40, 0x53, 0x3a, 0xdb, 0x0d, 0xea, 0x0b, 0xba, - 0x61, 0x51, 0x43, 0x85, 0x4c, 0x0d, 0x1d, 0xa6, 0x09, 0x29, 0x4a, 0x2b, 0xcf, 0xd0, 0x4b, 0x92, - 0x84, 0xea, 0x50, 0x0c, 0x03, 0x67, 0x3c, 0xf6, 0x6b, 0x25, 0xee, 0xe6, 0x77, 0x96, 0x65, 0x07, - 0x41, 0xa7, 0x63, 0x35, 0x45, 0x59, 0x14, 0xc2, 0xa0, 0x33, 0xf6, 0x8d, 0xd7, 0x50, 0xc1, 0xe1, - 0x43, 0xfb, 0x96, 0x3b, 0x60, 0x40, 0xf1, 0x9a, 0x4c, 0xc2, 0x88, 0xc8, 0xca, 0x02, 0x39, 0x79, - 0x71, 0xf8, 0x80, 0xe5, 0x0e, 0xda, 0x83, 0x82, 0x3b, 0x49, 0x8a, 0x23, 0x2b, 0x22, 0x36, 0x0c, - 0x17, 0xca, 0x38, 0x7c, 0xe0, 0x79, 0x42, 0x5f, 0x80, 0x40, 0xc4, 0x09, 0xdc, 0x69, 0x02, 0x77, - 0x85, 0x73, 0xfa, 0xee, 0x94, 0xa0, 0x57, 0xa0, 0x45, 0xe1, 0x83, 0x33, 0xe2, 0xc7, 0x8b, 0xd6, - 0xd1, 0x1a, 0xdb, 0x99, 0x6a, 0x4a, 0x9c, 0xc3, 0x10, 0x25, 0x64, 0x6c, 0xbc, 0x06, 0x58, 0x14, - 0xc3, 0xc7, 0x0e, 0xf9, 0x01, 0x83, 0x8f, 0xf8, 0xe3, 0xc4, 0xfe, 0xaa, 0x74, 0x99, 0x5b, 0xc0, - 0x72, 0x8f, 0x01, 0x31, 0x64, 0xd9, 0x3e, 0xa1, 0xde, 0xf8, 0x33, 0x6a, 0x04, 0x81, 0x7a, 0x43, - 0xbd, 0x31, 0x2f, 0x8e, 0x0a, 0xe6, 0xb4, 0xf1, 0x2d, 0x14, 0xae, 0xb8, 0xb9, 0x57, 0xa0, 0x71, - 0x29, 0x87, 0xb1, 0x93, 0xa6, 0xc9, 0x84, 0x99, 0x1e, 0x8d, 0x21, 0x4e, 0xc8, 0xd8, 0x68, 0xc2, - 0xda, 0xb9, 0x3c, 0x96, 0x0b, 0x7c, 0xba, 0x5f, 0xc6, 0x5f, 0x72, 0x50, 0x3a, 0x0b, 0xe7, 0x51, - 0xe0, 0xfa, 0xa8, 0x0a, 0x39, 0x6f, 0xcc, 0xf5, 0xf2, 0x38, 0xe7, 0x8d, 0xd1, 0xaf, 0xa0, 0x3a, - 0xf5, 0x6e, 0x22, 0x97, 0xd5, 0x83, 0x28, 0x6d, 0xd1, 0x9d, 0xdf, 0x5d, 0xf6, 0xac, 0x97, 0x48, - 0xf0, 0xfa, 0x5e, 0x9b, 0x2e, 0x2f, 0x97, 0x2a, 0x36, 0x9f, 0xa9, 0xd8, 0xaf, 0xa0, 0xea, 0x87, - 0x23, 0xd7, 0x77, 0xd2, 0x79, 0xa9, 0x72, 0xa7, 0xd6, 0x38, 0xf7, 0x22, 0x19, 0x9a, 0x4f, 0x70, - 0x29, 0xbc, 0x10, 0x17, 0xf4, 0x0d, 0xac, 0xce, 0xdc, 0x88, 0x7a, 0x23, 0x6f, 0xe6, 0xb2, 0x2f, - 0x8e, 0x22, 0x57, 0xcc, 0xb8, 0x9d, 0xc1, 0x0d, 0x67, 0xc4, 0xd1, 0xd7, 0xa0, 0xc7, 0x7c, 0x16, - 0x38, 0x0f, 0x61, 0x74, 0x37, 0xf1, 0xc3, 0x87, 0xb8, 0x56, 0xe2, 0xfe, 0xaf, 0x0b, 0xfe, 0x9b, - 0x84, 0x6d, 0xfc, 0x33, 0x07, 0xc5, 0x2b, 0x51, 0x65, 0x87, 0xa0, 0x72, 0x8c, 0xc4, 0x57, 0xc5, - 0xce, 0xf2, 0x61, 0x42, 0x82, 0x03, 0xc4, 0x65, 0xd0, 0xf7, 0xa0, 0x42, 0xbd, 0x29, 0x89, 0xa9, - 0x3b, 0x9d, 0x71, 0x50, 0xf3, 0x78, 0xc1, 0x78, 0x5f, 0xad, 0xb0, 0x4f, 0x07, 0xd6, 0xb4, 0x02, - 0x26, 0x46, 0xa2, 0x9f, 0x40, 0x85, 0xf5, 0x06, 0xff, 0xd2, 0xa9, 0x15, 0x78, 0xb3, 0x6d, 0x3d, - 0xe9, 0x0c, 0x7e, 0x2c, 0x2e, 0x47, 0x49, 0xb7, 0xfd, 0x02, 0x34, 0x5e, 0xcd, 0x52, 0x49, 0x4c, - 0x8b, 0x9d, 0xec, 0xb4, 0x48, 0xba, 0x06, 0xc3, 0x62, 0xc0, 0xa2, 0x7d, 0x28, 0xdc, 0x73, 0x97, - 0x4a, 0xf2, 0x8b, 0x6b, 0x39, 0x38, 0x0e, 0xbf, 0xd8, 0x67, 0xd7, 0xd9, 0x6f, 0x44, 0x35, 0xd5, - 0xca, 0xcf, 0xaf, 0x33, 0x59, 0x68, 0x38, 0x91, 0x41, 0x5f, 0xc2, 0xea, 0x68, 0x1e, 0x45, 0xfc, - 0x8b, 0xce, 0x9b, 0x92, 0xda, 0x16, 0x87, 0x42, 0x93, 0x3c, 0xdb, 0x9b, 0x12, 0xe3, 0xf7, 0x39, - 0xa8, 0x5e, 0x89, 0x3b, 0x2f, 0xb9, 0x67, 0xbf, 0x85, 0x4d, 0x32, 0x99, 0x90, 0x11, 0xf5, 0xee, - 0x89, 0x33, 0x72, 0x7d, 0x9f, 0x44, 0x8e, 0x2c, 0x5c, 0xad, 0xb1, 0x5e, 0x17, 0xdf, 0xbe, 0x6d, - 0xce, 0xef, 0x76, 0xf0, 0x46, 0x2a, 0x2b, 0x59, 0x63, 0x64, 0xc2, 0xa6, 0x37, 0x9d, 0x92, 0xb1, - 0xe7, 0xd2, 0x65, 0x03, 0x62, 0x62, 0x6d, 0xcb, 0xf6, 0xbf, 0xb2, 0x4f, 0x5c, 0x4a, 0x16, 0x66, - 0x52, 0x8d, 0xd4, 0xcc, 0x57, 0xac, 0xba, 0xa3, 0x9b, 0xf4, 0xea, 0x5e, 0x93, 0x9a, 0x36, 0x67, - 0x62, 0xb9, 0x99, 0xf9, 0x2c, 0x50, 0x9f, 0x7c, 0x16, 0x2c, 0x46, 0x77, 0xe1, 0x63, 0xa3, 0xdb, - 0xf8, 0x06, 0xd6, 0x53, 0x20, 0xe4, 0xb5, 0x7f, 0x08, 0x45, 0x9e, 0xca, 0x64, 0x66, 0xa0, 0xe7, - 0x55, 0x87, 0xa5, 0x84, 0xf1, 0xbb, 0x1c, 0xa0, 0x44, 0x3f, 0x7c, 0x88, 0xff, 0x47, 0xc1, 0xdc, - 0x82, 0x02, 0xe7, 0x4b, 0x24, 0xc5, 0x82, 0xe1, 0xe0, 0xbb, 0x31, 0x9d, 0xdd, 0xa5, 0x30, 0x0a, - 0xe5, 0xd7, 0xec, 0x17, 0x93, 0x78, 0xee, 0x53, 0x2c, 0x25, 0x8c, 0xbf, 0x2a, 0xb0, 0x99, 0xc1, - 0x41, 0x62, 0xb9, 0xb8, 0x06, 0x94, 0xff, 0x7c, 0x0d, 0xa0, 0x03, 0x28, 0xcf, 0xee, 0x3e, 0x70, - 0x5d, 0xa4, 0xbb, 0xef, 0xed, 0xe2, 0xef, 0x83, 0x1a, 0xb1, 0x69, 0xa2, 0x72, 0xcd, 0xe5, 0xbb, - 0x91, 0xf3, 0xd9, 0x05, 0x9b, 0x89, 0x23, 0x73, 0xc1, 0x4a, 0xff, 0xff, 0xa1, 0xc0, 0xf6, 0xa2, - 0x0e, 0xe6, 0x3e, 0xfd, 0xbf, 0x4a, 0xa5, 0x11, 0xc1, 0xce, 0xd3, 0xe8, 0x3e, 0x29, 0x41, 0x9f, - 0x01, 0xfb, 0xe1, 0x2f, 0x41, 0x5b, 0xfa, 0xf4, 0x61, 0x2f, 0xa4, 0xee, 0x49, 0x7f, 0x80, 0x4d, - 0x7d, 0x05, 0x95, 0x41, 0x1d, 0xda, 0x83, 0x0b, 0x5d, 0x61, 0x94, 0xf9, 0x6b, 0xb3, 0x2d, 0x5e, - 0x5d, 0x8c, 0x72, 0xa4, 0x50, 0xfe, 0xf0, 0xef, 0x0a, 0xc0, 0x62, 0xc6, 0x23, 0x0d, 0x4a, 0x97, - 0xfd, 0xf3, 0xfe, 0xe0, 0x4d, 0x5f, 0x18, 0x38, 0xb1, 0xbb, 0x1d, 0x5d, 0x41, 0x15, 0x28, 0x88, - 0x67, 0x5c, 0x8e, 0x9d, 0x20, 0xdf, 0x70, 0x79, 0xf6, 0xc0, 0x4b, 0x1f, 0x70, 0x2a, 0x2a, 0x41, - 0x3e, 0x7d, 0xa6, 0xc9, 0x77, 0x59, 0x91, 0x19, 0xc4, 0xe6, 0x85, 0xd5, 0x6c, 0x9b, 0x7a, 0x89, - 0x6d, 0xa4, 0x2f, 0x34, 0x80, 0x62, 0xf2, 0x3c, 0x63, 0x9a, 0xec, 0x51, 0x07, 0xec, 0x9c, 0x81, - 0x7d, 0x6a, 0x62, 0x5d, 0x63, 0x3c, 0x3c, 0x78, 0xa3, 0xaf, 0x32, 0xde, 0x71, 0xd7, 0xb4, 0x3a, - 0xfa, 0x1a, 0x7b, 0xd5, 0x9d, 0x9a, 0x4d, 0x6c, 0xb7, 0xcc, 0xa6, 0xad, 0x57, 0xd9, 0xce, 0x15, - 0x77, 0x70, 0x9d, 0x1d, 0x73, 0x36, 0xb8, 0xc4, 0xfd, 0xa6, 0xa5, 0xeb, 0x87, 0xfb, 0xb0, 0x96, - 0xb9, 0xda, 0xd9, 0x59, 0x76, 0xb3, 0x65, 0x99, 0x43, 0x7d, 0x85, 0xd1, 0xc3, 0xd3, 0x26, 0xee, - 0x0c, 0x75, 0xa5, 0xf5, 0xf5, 0xdb, 0xfd, 0x7b, 0x8f, 0x92, 0x38, 0xae, 0x7b, 0xe1, 0x91, 0xa0, - 0x8e, 0x6e, 0xc2, 0xa3, 0x7b, 0x7a, 0xc4, 0xff, 0x61, 0x38, 0x5a, 0x4c, 0xa4, 0xeb, 0x22, 0xe7, - 0xfc, 0xf4, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe4, 0x09, 0xf3, 0xd7, 0xbd, 0x10, 0x00, 0x00, + // 1680 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0x4b, 0x73, 0x23, 0x49, + 0x11, 0x9e, 0x96, 0x5a, 0xaf, 0x6c, 0x5b, 0x6e, 0x97, 0x1f, 0x88, 0x09, 0x96, 0xf0, 0x76, 0x30, + 0x8c, 0xd7, 0x11, 0xc8, 0x20, 0x60, 0x38, 0x2d, 0x8b, 0x1e, 0x6d, 0x8f, 0x66, 0x5a, 0x92, 0xa7, + 0xd4, 0xf6, 0x10, 0x7b, 0xe9, 0x68, 0x4b, 0x25, 0xbb, 0x71, 0x3f, 0x34, 0xdd, 0x25, 0x7b, 0xf5, + 0x03, 0x08, 0x7e, 0x00, 0xbf, 0x82, 0x33, 0x57, 0x38, 0x11, 0xc1, 0x9d, 0x3b, 0x57, 0x7e, 0x00, + 0xff, 0x80, 0xa8, 0x47, 0xb7, 0xd4, 0x9a, 0x65, 0xc7, 0xb3, 0x11, 0x1c, 0xd8, 0x8b, 0x22, 0x2b, + 0x2b, 0x33, 0x2b, 0xf3, 0xcb, 0x47, 0x75, 0x09, 0xf4, 0x6b, 0x2f, 0xf4, 0xa3, 0x9b, 0xa9, 0x4b, + 0xdd, 0xe6, 0x3c, 0x8e, 0x68, 0x84, 0x60, 0xc5, 0x79, 0xaa, 0xdd, 0xd3, 0x78, 0x3e, 0x11, 0x1b, + 0x4f, 0xb5, 0x77, 0x0b, 0x12, 0x2f, 0xe5, 0xa2, 0x4e, 0xa3, 0x79, 0xb4, 0xd2, 0x32, 0x06, 0x50, + 0xe9, 0xde, 0xba, 0x71, 0x42, 0x28, 0x3a, 0x84, 0xf2, 0xc4, 0xf7, 0x48, 0x48, 0x1b, 0xca, 0x91, + 0x72, 0x5c, 0xc2, 0x72, 0x85, 0x10, 0xa8, 0x93, 0x28, 0x0c, 0x1b, 0x05, 0xce, 0xe5, 0x34, 0x93, + 0x4d, 0x48, 0x7c, 0x4f, 0xe2, 0x46, 0x51, 0xc8, 0x8a, 0x95, 0xf1, 0xaf, 0x22, 0xec, 0x76, 0xb8, + 0x1f, 0x76, 0xec, 0x86, 0x89, 0x3b, 0xa1, 0x5e, 0x14, 0xa2, 0x73, 0x80, 0x84, 0xba, 0x94, 0x04, + 0x24, 0xa4, 0x49, 0x43, 0x39, 0x2a, 0x1e, 0x6b, 0xad, 0xe7, 0xcd, 0xb5, 0x08, 0xde, 0x53, 0x69, + 0x8e, 0x53, 0x79, 0xbc, 0xa6, 0x8a, 0x5a, 0xa0, 0x91, 0x7b, 0x12, 0x52, 0x87, 0x46, 0x77, 0x24, + 0x6c, 0xa8, 0x47, 0xca, 0xb1, 0xd6, 0xda, 0x6d, 0x8a, 0x00, 0x4d, 0xb6, 0x63, 0xb3, 0x0d, 0x0c, + 0x24, 0xa3, 0x9f, 0xfe, 0xbd, 0x00, 0xb5, 0xcc, 0x1a, 0xb2, 0xa0, 0x3a, 0x71, 0x29, 0xb9, 0x89, + 0xe2, 0x25, 0x0f, 0xb3, 0xde, 0xfa, 0xe9, 0x23, 0x1d, 0x69, 0x76, 0xa5, 0x1e, 0xce, 0x2c, 0xa0, + 0x9f, 0x40, 0x65, 0x22, 0xd0, 0xe3, 0xe8, 0x68, 0xad, 0xbd, 0x75, 0x63, 0x12, 0x58, 0x9c, 0xca, + 0x20, 0x1d, 0x8a, 0xc9, 0x3b, 0x9f, 0x43, 0xb6, 0x85, 0x19, 0x69, 0xfc, 0x49, 0x81, 0x6a, 0x6a, + 0x17, 0xed, 0xc1, 0x4e, 0xc7, 0x72, 0x2e, 0x87, 0xd8, 0xec, 0x8e, 0xce, 0x87, 0xfd, 0x2f, 0xcd, + 0x9e, 0xfe, 0x04, 0x6d, 0x41, 0xb5, 0x63, 0x39, 0x1d, 0xf3, 0xbc, 0x3f, 0xd4, 0x15, 0xb4, 0x0d, + 0xb5, 0x8e, 0xe5, 0x74, 0x47, 0x83, 0x41, 0xdf, 0xd6, 0x0b, 0x68, 0x07, 0xb4, 0x8e, 0xe5, 0xe0, + 0x91, 0x65, 0x75, 0xda, 0xdd, 0xd7, 0x7a, 0x11, 0x1d, 0xc0, 0x6e, 0xc7, 0x72, 0x7a, 0x03, 0xcb, + 0xe9, 0x99, 0x17, 0xd8, 0xec, 0xb6, 0x6d, 0xb3, 0xa7, 0xab, 0x08, 0xa0, 0xcc, 0xd8, 0x3d, 0x4b, + 0x2f, 0x49, 0x7a, 0x6c, 0xda, 0x7a, 0x59, 0x9a, 0xeb, 0x0f, 0xc7, 0x26, 0xb6, 0xf5, 0x8a, 0x5c, + 0x5e, 0x5e, 0xf4, 0xda, 0xb6, 0xa9, 0x57, 0xe5, 0xb2, 0x67, 0x5a, 0xa6, 0x6d, 0xea, 0xb5, 0x57, + 0x6a, 0xb5, 0xa0, 0x17, 0x5f, 0xa9, 0xd5, 0xa2, 0xae, 0x1a, 0x7f, 0x54, 0xe0, 0x60, 0x4c, 0x63, + 0xe2, 0x06, 0xaf, 0xc9, 0x12, 0xbb, 0xe1, 0x0d, 0xc1, 0xe4, 0xdd, 0x82, 0x24, 0x14, 0x3d, 0x85, + 0xea, 0x3c, 0x4a, 0x3c, 0x86, 0x1d, 0x07, 0xb8, 0x86, 0xb3, 0x35, 0x3a, 0x85, 0xda, 0x1d, 0x59, + 0x3a, 0x31, 0x93, 0x97, 0x80, 0xa1, 0x66, 0x56, 0x90, 0x99, 0xa5, 0xea, 0x9d, 0xa4, 0xd6, 0xf1, + 0x2d, 0x7e, 0x18, 0x5f, 0x63, 0x06, 0x87, 0x9b, 0x4e, 0x25, 0xf3, 0x28, 0x4c, 0x08, 0xb2, 0x00, + 0x09, 0x45, 0x87, 0xae, 0x72, 0xcb, 0xfd, 0xd3, 0x5a, 0x9f, 0x7c, 0x63, 0x01, 0xe0, 0xdd, 0xeb, + 0x4d, 0x96, 0xf1, 0x15, 0xec, 0x89, 0x73, 0x6c, 0xf7, 0xda, 0x27, 0xc9, 0x63, 0x42, 0x3f, 0x84, + 0x32, 0xe5, 0xc2, 0x8d, 0xc2, 0x51, 0xf1, 0xb8, 0x86, 0xe5, 0xea, 0x63, 0x23, 0x9c, 0xc2, 0x7e, + 0xfe, 0xe4, 0xff, 0x49, 0x7c, 0xbf, 0x00, 0x15, 0x2f, 0x7c, 0x82, 0xf6, 0xa1, 0x14, 0xb8, 0x74, + 0x72, 0x2b, 0xa3, 0x11, 0x0b, 0x16, 0xca, 0xcc, 0xf3, 0x29, 0x89, 0x79, 0x0a, 0x6b, 0x58, 0xae, + 0x8c, 0x3f, 0x2b, 0x50, 0x3e, 0xe3, 0x24, 0xfa, 0x31, 0x94, 0xe2, 0x05, 0x0b, 0x56, 0xf4, 0xba, + 0xbe, 0xee, 0x01, 0xb3, 0x8c, 0xc5, 0x36, 0xea, 0x43, 0x7d, 0xe6, 0x11, 0x7f, 0xca, 0x5b, 0x77, + 0x10, 0x4d, 0x45, 0x55, 0xd4, 0x5b, 0x9f, 0xae, 0x2b, 0x08, 0x9b, 0xcd, 0xb3, 0x9c, 0x20, 0xde, + 0x50, 0x34, 0x5e, 0x40, 0x3d, 0x2f, 0xc1, 0xda, 0xc9, 0xc4, 0xd8, 0x19, 0x0d, 0x9d, 0x41, 0x7f, + 0x3c, 0x68, 0xdb, 0xdd, 0x97, 0xfa, 0x13, 0xde, 0x31, 0xe6, 0xd8, 0x76, 0xcc, 0xb3, 0xb3, 0x11, + 0xb6, 0x75, 0xc5, 0xf8, 0x5b, 0x01, 0xb6, 0x04, 0x28, 0xe3, 0x68, 0x11, 0x4f, 0x08, 0xcb, 0xe2, + 0x1d, 0x59, 0x26, 0x73, 0x77, 0x42, 0xd2, 0x2c, 0xa6, 0x6b, 0x06, 0x48, 0x72, 0xeb, 0xc6, 0x53, + 0x19, 0xb9, 0x58, 0xa0, 0x5f, 0x82, 0xc6, 0xb3, 0x49, 0x1d, 0xba, 0x9c, 0x13, 0x9e, 0xc7, 0x7a, + 0x6b, 0x7f, 0x55, 0xd8, 0x3c, 0x57, 0xd4, 0x5e, 0xce, 0x09, 0x06, 0x9a, 0xd1, 0xf9, 0x6e, 0x50, + 0x1f, 0xd1, 0x0d, 0xab, 0x1a, 0x2a, 0xe5, 0x6a, 0xe8, 0x24, 0x4b, 0x48, 0x59, 0x5a, 0x79, 0x0f, + 0xbd, 0x34, 0x49, 0xa8, 0x09, 0xe5, 0x28, 0x74, 0xa6, 0x53, 0xbf, 0x51, 0xe1, 0x6e, 0x7e, 0x6f, + 0x5d, 0x76, 0x14, 0xf6, 0x7a, 0x56, 0x5b, 0x94, 0x45, 0x29, 0x0a, 0x7b, 0x53, 0x1f, 0x3d, 0x83, + 0x3a, 0xf9, 0x8a, 0x92, 0x38, 0x74, 0x7d, 0x27, 0x58, 0xb2, 0xe9, 0x55, 0xe5, 0xa1, 0x6f, 0xa7, + 0xdc, 0x01, 0x63, 0x1a, 0x6f, 0xa0, 0x86, 0xa3, 0x87, 0xee, 0x2d, 0xf7, 0xd3, 0x80, 0xf2, 0x35, + 0x99, 0x45, 0x31, 0x91, 0x05, 0x08, 0x72, 0x40, 0xe3, 0xe8, 0x01, 0xcb, 0x1d, 0x74, 0x04, 0x25, + 0x77, 0x96, 0xd6, 0x50, 0x5e, 0x44, 0x6c, 0x18, 0x2e, 0x54, 0x71, 0xf4, 0xc0, 0xd3, 0x89, 0x3e, + 0x01, 0x01, 0x9c, 0x13, 0xba, 0x41, 0x9a, 0x95, 0x1a, 0xe7, 0x0c, 0xdd, 0x80, 0xa0, 0x17, 0xa0, + 0xc5, 0xd1, 0x83, 0x33, 0xe1, 0xc7, 0x8b, 0x0e, 0xd3, 0x5a, 0x07, 0xb9, 0xa2, 0x4b, 0x9d, 0xc3, + 0x10, 0xa7, 0x64, 0x62, 0xbc, 0x01, 0x58, 0xd5, 0xcc, 0x87, 0x0e, 0xf9, 0x11, 0x43, 0x99, 0xf8, + 0xd3, 0xd4, 0xfe, 0x96, 0x74, 0x99, 0x5b, 0xc0, 0x72, 0x8f, 0x01, 0x31, 0x66, 0x45, 0x71, 0x4e, + 0xbd, 0xe9, 0xb7, 0x28, 0x25, 0x04, 0xea, 0x0d, 0xf5, 0xa6, 0xbc, 0x86, 0x6a, 0x98, 0xd3, 0xc6, + 0x17, 0x50, 0xba, 0xe2, 0xe6, 0x5e, 0x80, 0xc6, 0xa5, 0x1c, 0xc6, 0x4e, 0x7b, 0x2b, 0x17, 0x66, + 0x76, 0x34, 0x86, 0x24, 0x25, 0x13, 0xa3, 0x0d, 0xdb, 0xaf, 0xe5, 0xb1, 0x5c, 0xe0, 0xe3, 0xfd, + 0x32, 0xfe, 0x52, 0x80, 0xca, 0xab, 0x68, 0xc1, 0x12, 0x8e, 0xea, 0x50, 0xf0, 0xa6, 0x5c, 0xaf, + 0x88, 0x0b, 0xde, 0x14, 0xfd, 0x06, 0xea, 0x81, 0x77, 0x13, 0xbb, 0xac, 0x6c, 0x44, 0x07, 0x88, + 0x26, 0xfe, 0xfe, 0xba, 0x67, 0x83, 0x54, 0x82, 0xb7, 0xc1, 0x76, 0xb0, 0xbe, 0x5c, 0x2b, 0xec, + 0x62, 0xae, 0xb0, 0x9f, 0x41, 0xdd, 0x8f, 0x26, 0xae, 0xef, 0x64, 0x63, 0x55, 0x15, 0xc5, 0xc7, + 0xb9, 0x17, 0xe9, 0x6c, 0xdd, 0xc0, 0xa5, 0xf4, 0x48, 0x5c, 0xd0, 0xe7, 0xb0, 0x35, 0x77, 0x63, + 0xea, 0x4d, 0xbc, 0xb9, 0xcb, 0x3e, 0x4c, 0xca, 0x5c, 0x31, 0xe7, 0x76, 0x0e, 0x37, 0x9c, 0x13, + 0x47, 0x9f, 0x81, 0x9e, 0xf0, 0x91, 0xe1, 0x3c, 0x44, 0xf1, 0xdd, 0xcc, 0x8f, 0x1e, 0x92, 0x46, + 0x85, 0xfb, 0xbf, 0x23, 0xf8, 0x6f, 0x53, 0xb6, 0xf1, 0xef, 0x02, 0x94, 0xaf, 0x44, 0x95, 0x9d, + 0x80, 0xca, 0x31, 0x12, 0x1f, 0x1f, 0x87, 0xeb, 0x87, 0x09, 0x09, 0x0e, 0x10, 0x97, 0x41, 0x3f, + 0x80, 0x1a, 0xf5, 0x02, 0x92, 0x50, 0x37, 0x98, 0x73, 0x50, 0x8b, 0x78, 0xc5, 0xf8, 0xba, 0x5a, + 0x61, 0x5f, 0x18, 0xac, 0xb7, 0x05, 0x4c, 0x8c, 0x44, 0x3f, 0x83, 0x1a, 0xeb, 0x0d, 0xfe, 0x41, + 0xd4, 0x28, 0xf1, 0x66, 0xdb, 0xdf, 0xe8, 0x0c, 0x7e, 0x2c, 0xae, 0xc6, 0x69, 0xb7, 0xfd, 0x0a, + 0x34, 0x5e, 0xcd, 0x52, 0x49, 0x0c, 0x95, 0xc3, 0xfc, 0x50, 0x49, 0xbb, 0x06, 0xc3, 0x6a, 0x0e, + 0xa3, 0xe7, 0x50, 0xba, 0xe7, 0x2e, 0x55, 0xe4, 0x87, 0xd9, 0x7a, 0x70, 0x1c, 0x7e, 0xb1, 0xcf, + 0x6e, 0xbd, 0xdf, 0x89, 0x6a, 0xe2, 0xe3, 0x64, 0xe3, 0xd6, 0x93, 0x85, 0x86, 0x53, 0x19, 0x1e, + 0x55, 0xe0, 0x37, 0x6a, 0x32, 0xaa, 0xc0, 0x47, 0x9f, 0xc2, 0xd6, 0x64, 0x11, 0xc7, 0xfc, 0x53, + 0xd0, 0x0b, 0x48, 0x63, 0x9f, 0x83, 0xa3, 0x49, 0x9e, 0xed, 0x05, 0xc4, 0xf8, 0x43, 0x01, 0xea, + 0x57, 0xe2, 0xb2, 0x4c, 0x2f, 0xe8, 0x2f, 0x60, 0x8f, 0xcc, 0x66, 0x64, 0x42, 0xbd, 0x7b, 0xe2, + 0x4c, 0x5c, 0xdf, 0x27, 0xb1, 0x23, 0x4b, 0x59, 0x6b, 0xed, 0x34, 0xc5, 0x47, 0x73, 0x97, 0xf3, + 0xfb, 0x3d, 0xbc, 0x9b, 0xc9, 0x4a, 0xd6, 0x14, 0x99, 0xb0, 0xe7, 0x05, 0x01, 0x99, 0x7a, 0x2e, + 0x5d, 0x37, 0x20, 0x66, 0xd8, 0x81, 0x1c, 0x08, 0x57, 0xf6, 0xb9, 0x4b, 0xc9, 0xca, 0x4c, 0xa6, + 0x91, 0x99, 0x79, 0xc6, 0xea, 0x3d, 0xbe, 0xc9, 0xee, 0xfc, 0x6d, 0xa9, 0x69, 0x73, 0x26, 0x96, + 0x9b, 0xb9, 0xef, 0x09, 0x75, 0xe3, 0x7b, 0x62, 0x35, 0xf3, 0x4b, 0x1f, 0x9a, 0xf9, 0xc6, 0xe7, + 0xb0, 0x93, 0x01, 0x21, 0xbf, 0x17, 0x4e, 0xa0, 0xcc, 0x93, 0x9b, 0x4e, 0x11, 0xf4, 0x7e, 0x1d, + 0x62, 0x29, 0x61, 0xfc, 0xbe, 0x00, 0x28, 0xd5, 0x8f, 0x1e, 0x92, 0xff, 0x53, 0x30, 0xf7, 0xa1, + 0xc4, 0xf9, 0x12, 0x49, 0xb1, 0x60, 0x38, 0xf8, 0x6e, 0x42, 0xe7, 0x77, 0x19, 0x8c, 0x42, 0xf9, + 0x0d, 0xfb, 0xc5, 0x24, 0x59, 0xf8, 0x14, 0x4b, 0x09, 0xe3, 0xaf, 0x0a, 0xec, 0xe5, 0x70, 0x90, + 0x58, 0xae, 0x2e, 0x06, 0xe5, 0xbf, 0x5f, 0x0c, 0xe8, 0x18, 0xaa, 0xf3, 0xbb, 0x6f, 0xb8, 0x40, + 0xb2, 0xdd, 0xaf, 0xed, 0xeb, 0x1f, 0x82, 0x1a, 0xb3, 0xf9, 0xa2, 0x72, 0xcd, 0xf5, 0xdb, 0x92, + 0xf3, 0xd9, 0x95, 0x9b, 0x8b, 0x23, 0x77, 0xe5, 0x4a, 0xff, 0xff, 0xa9, 0xc0, 0xc1, 0xaa, 0x0e, + 0x16, 0x3e, 0xfd, 0x4e, 0xa5, 0xd2, 0x88, 0xe1, 0x70, 0x33, 0xba, 0x8f, 0x4a, 0xd0, 0xb7, 0x80, + 0xfd, 0xe4, 0xd7, 0xa0, 0xad, 0x7d, 0x33, 0xb1, 0xa7, 0x55, 0xff, 0x7c, 0x38, 0xc2, 0xa6, 0xfe, + 0x04, 0x55, 0x41, 0x1d, 0xdb, 0xa3, 0x0b, 0x5d, 0x61, 0x94, 0xf9, 0x5b, 0xb3, 0x2b, 0x9e, 0x6b, + 0x8c, 0x72, 0xa4, 0x50, 0xf1, 0xe4, 0x1f, 0x0a, 0xc0, 0x6a, 0xea, 0x23, 0x0d, 0x2a, 0x97, 0xc3, + 0xd7, 0xc3, 0xd1, 0xdb, 0xa1, 0x30, 0x70, 0x6e, 0xf7, 0x7b, 0xba, 0x82, 0x6a, 0x50, 0x12, 0xef, + 0xbf, 0x02, 0x3b, 0x41, 0x3e, 0xfe, 0x8a, 0xec, 0x65, 0x98, 0xbd, 0xfc, 0x54, 0x54, 0x81, 0x62, + 0xf6, 0xbe, 0x93, 0x0f, 0xba, 0x32, 0x33, 0x88, 0xcd, 0x0b, 0xab, 0xdd, 0x35, 0xf5, 0x0a, 0xdb, + 0xc8, 0x9e, 0x76, 0x00, 0xe5, 0xf4, 0x5d, 0xc7, 0x34, 0xd9, 0x6b, 0x10, 0xd8, 0x39, 0x23, 0xfb, + 0xa5, 0x89, 0x75, 0x8d, 0xf1, 0xf0, 0xe8, 0xad, 0xbe, 0xc5, 0x78, 0x67, 0x7d, 0xd3, 0xea, 0xe9, + 0xdb, 0xec, 0x39, 0xf8, 0xd2, 0x6c, 0x63, 0xbb, 0x63, 0xb6, 0x6d, 0xbd, 0xce, 0x76, 0xae, 0xb8, + 0x83, 0x3b, 0xec, 0x98, 0x57, 0xa3, 0x4b, 0x3c, 0x6c, 0x5b, 0xba, 0x7e, 0xf2, 0x1c, 0xb6, 0x73, + 0x97, 0x3d, 0x3b, 0xcb, 0x6e, 0x77, 0x2c, 0x73, 0xac, 0x3f, 0x61, 0xf4, 0xf8, 0x65, 0x1b, 0xf7, + 0xc6, 0xba, 0xd2, 0xf9, 0xec, 0xcb, 0xe7, 0xf7, 0x1e, 0x25, 0x49, 0xd2, 0xf4, 0xa2, 0x53, 0x41, + 0x9d, 0xde, 0x44, 0xa7, 0xf7, 0xf4, 0x94, 0xff, 0x35, 0x71, 0xba, 0x9a, 0x48, 0xd7, 0x65, 0xce, + 0xf9, 0xf9, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x91, 0xae, 0x28, 0x7d, 0xf6, 0x10, 0x00, 0x00, } diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go index fd1f76410e8..52c75655d2e 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go @@ -5,8 +5,9 @@ package tabletmanagerdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" logutil "vitess.io/vitess/go/vt/proto/logutil" query "vitess.io/vitess/go/vt/proto/query" replicationdata "vitess.io/vitess/go/vt/proto/replicationdata" diff --git a/go/vt/servenv/buildinfo.go b/go/vt/servenv/buildinfo.go index 97834a794d8..cab58cf2555 100644 --- a/go/vt/servenv/buildinfo.go +++ b/go/vt/servenv/buildinfo.go @@ -96,4 +96,14 @@ func init() { stats.NewString("GoOS").Set(AppVersion.goOS) stats.NewString("GoArch").Set(AppVersion.goArch) + buildLabels := []string{"BuildHost", "BuildUser", "BuildTimestamp", "BuildGitRev", "BuildGitBranch", "BuildNumber"} + buildValues := []string{ + AppVersion.buildHost, + AppVersion.buildUser, + fmt.Sprintf("%v", AppVersion.buildTime), + AppVersion.buildGitRev, + AppVersion.buildGitBranch, + fmt.Sprintf("%v", AppVersion.jenkinsBuildNumber), + } + stats.NewGaugesWithMultiLabels("BuildInformation", "build information exposed via label", buildLabels).Set(buildValues, 1) } diff --git a/go/vt/srvtopo/resilient_server.go b/go/vt/srvtopo/resilient_server.go index dce6f3dccda..efd126859e0 100644 --- a/go/vt/srvtopo/resilient_server.go +++ b/go/vt/srvtopo/resilient_server.go @@ -211,11 +211,19 @@ func NewResilientServer(base *topo.Server, counterPrefix string) *ResilientServe log.Fatalf("srv_topo_cache_refresh must be less than or equal to srv_topo_cache_ttl") } + var metric string + + if counterPrefix == "" { + metric = counterPrefix + "Counts" + } else { + metric = "" + } + return &ResilientServer{ topoServer: base, cacheTTL: *srvTopoCacheTTL, cacheRefresh: *srvTopoCacheRefresh, - counts: stats.NewCountersWithSingleLabel(counterPrefix+"Counts", "Resilient srvtopo server operations", "type"), + counts: stats.NewCountersWithSingleLabel(metric, "Resilient srvtopo server operations", "type"), srvKeyspaceNamesCache: make(map[string]*srvKeyspaceNamesEntry), srvKeyspaceCache: make(map[string]*srvKeyspaceEntry), diff --git a/go/vt/tlstest/tlstest_test.go b/go/vt/tlstest/tlstest_test.go index 112fd493c41..1f22403cddd 100644 --- a/go/vt/tlstest/tlstest_test.go +++ b/go/vt/tlstest/tlstest_test.go @@ -18,6 +18,7 @@ package tlstest import ( "crypto/tls" + "crypto/x509" "fmt" "io" "io/ioutil" @@ -29,6 +30,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "vitess.io/vitess/go/vt/vttls" ) @@ -45,26 +47,20 @@ func TestClientServer(t *testing.T) { } defer os.RemoveAll(root) - // Create the certs and configs. - CreateCA(root) - - CreateSignedCert(root, CA, "01", "servers", "Servers CA") - CreateSignedCert(root, "servers", "01", "server-instance", "server.example.com") + clientServerKeyPairs := createClientServerCertPairs(root) - CreateSignedCert(root, CA, "02", "clients", "Clients CA") - CreateSignedCert(root, "clients", "01", "client-instance", "Client Instance") serverConfig, err := vttls.ServerConfig( - path.Join(root, "server-instance-cert.pem"), - path.Join(root, "server-instance-key.pem"), - path.Join(root, "clients-cert.pem")) + clientServerKeyPairs.serverCert, + clientServerKeyPairs.serverKey, + clientServerKeyPairs.clientCA) if err != nil { t.Fatalf("TLSServerConfig failed: %v", err) } clientConfig, err := vttls.ClientConfig( - path.Join(root, "client-instance-cert.pem"), - path.Join(root, "client-instance-key.pem"), - path.Join(root, "servers-cert.pem"), - "server.example.com") + clientServerKeyPairs.clientCert, + clientServerKeyPairs.clientKey, + clientServerKeyPairs.serverCA, + clientServerKeyPairs.serverName) if err != nil { t.Fatalf("TLSClientConfig failed: %v", err) } @@ -121,10 +117,10 @@ func TestClientServer(t *testing.T) { // badClientConfig, err := vttls.ClientConfig( - path.Join(root, "server-instance-cert.pem"), - path.Join(root, "server-instance-key.pem"), - path.Join(root, "servers-cert.pem"), - "server.example.com") + clientServerKeyPairs.serverCert, + clientServerKeyPairs.serverKey, + clientServerKeyPairs.serverCA, + clientServerKeyPairs.serverName) if err != nil { t.Fatalf("TLSClientConfig failed: %v", err) } @@ -168,3 +164,127 @@ func TestClientServer(t *testing.T) { t.Errorf("Wrong error returned: %v", err) } } + +var serialCounter = 0 + +type clientServerKeyPairs struct { + serverCert string + serverKey string + serverCA string + serverName string + clientCert string + clientKey string + clientCA string +} + +func createClientServerCertPairs(root string) clientServerKeyPairs { + + // Create the certs and configs. + CreateCA(root) + + serverSerial := fmt.Sprintf("%03d", serialCounter*2+1) + clientSerial := fmt.Sprintf("%03d", serialCounter*2+2) + + serialCounter = serialCounter + 1 + + serverName := fmt.Sprintf("server-%s", serverSerial) + serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial) + serverCertName := fmt.Sprintf("server-instance-%s", serverSerial) + serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial) + + clientName := fmt.Sprintf("clients-%s", serverSerial) + clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial) + clientCertName := fmt.Sprintf("client-instance-%s", serverSerial) + clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial) + + CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName) + CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName) + + CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName) + CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName) + + return clientServerKeyPairs{ + serverCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)), + serverKey: path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)), + serverCA: path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)), + clientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)), + clientKey: path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)), + clientCA: path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)), + serverName: serverCertCommonName, + } + +} + +func getServerConfig(keypairs clientServerKeyPairs) (*tls.Config, error) { + return vttls.ServerConfig( + keypairs.clientCert, + keypairs.clientKey, + keypairs.serverCA) +} + +func getClientConfig(keypairs clientServerKeyPairs) (*tls.Config, error) { + return vttls.ClientConfig( + keypairs.clientCert, + keypairs.clientKey, + keypairs.serverCA, + keypairs.serverName) +} + +func TestServerTLSConfigCaching(t *testing.T) { + testConfigGeneration(t, "servertlstest", getServerConfig, func(config *tls.Config) *x509.CertPool { + return config.ClientCAs + }) +} + +func TestClientTLSConfigCaching(t *testing.T) { + testConfigGeneration(t, "clienttlstest", getClientConfig, func(config *tls.Config) *x509.CertPool { + return config.RootCAs + }) +} + +func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(clientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) { + // Our test root. + root, err := ioutil.TempDir("", rootPrefix) + if err != nil { + t.Fatalf("TempDir failed: %v", err) + } + defer os.RemoveAll(root) + + const configsToGenerate = 1 + + firstClientServerKeyPairs := createClientServerCertPairs(root) + secondClientServerKeyPairs := createClientServerCertPairs(root) + + firstExpectedConfig, _ := generateConfig(firstClientServerKeyPairs) + secondExpectedConfig, _ := generateConfig(secondClientServerKeyPairs) + firstConfigChannel := make(chan *tls.Config, configsToGenerate) + secondConfigChannel := make(chan *tls.Config, configsToGenerate) + + var configCounter = 0 + + for i := 1; i <= configsToGenerate; i++ { + go func() { + firstConfig, _ := generateConfig(firstClientServerKeyPairs) + firstConfigChannel <- firstConfig + secondConfig, _ := generateConfig(secondClientServerKeyPairs) + secondConfigChannel <- secondConfig + }() + } + + for { + select { + case firstConfig := <-firstConfigChannel: + assert.Equal(t, &firstExpectedConfig.Certificates, &firstConfig.Certificates) + assert.Equal(t, getCertPool(firstExpectedConfig), getCertPool(firstConfig)) + case secondConfig := <-secondConfigChannel: + assert.Equal(t, &secondExpectedConfig.Certificates, &secondConfig.Certificates) + assert.Equal(t, getCertPool(secondExpectedConfig), getCertPool(secondConfig)) + } + configCounter = configCounter + 1 + + if configCounter >= 2*configsToGenerate { + break + } + } + +} diff --git a/go/vt/topo/keyspace.go b/go/vt/topo/keyspace.go index a22d2b4f5f4..7eb63e19f96 100755 --- a/go/vt/topo/keyspace.go +++ b/go/vt/topo/keyspace.go @@ -18,7 +18,6 @@ package topo import ( "path" - "sync" "github.com/golang/protobuf/proto" "golang.org/x/net/context" @@ -26,7 +25,6 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/event" - "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo/events" @@ -228,30 +226,16 @@ func (ts *Server) FindAllShardsInKeyspace(ctx context.Context, keyspace string) } result := make(map[string]*ShardInfo, len(shards)) - wg := sync.WaitGroup{} - mu := sync.Mutex{} - rec := concurrency.FirstErrorRecorder{} for _, shard := range shards { - wg.Add(1) - go func(shard string) { - defer wg.Done() - si, err := ts.GetShard(ctx, keyspace, shard) - if err != nil { - if IsErrType(err, NoNode) { - log.Warningf("GetShard(%v, %v) returned ErrNoNode, consider checking the topology.", keyspace, shard) - } else { - rec.RecordError(vterrors.Wrapf(err, "GetShard(%v, %v) failed", keyspace, shard)) - } - return + si, err := ts.GetShard(ctx, keyspace, shard) + if err != nil { + if IsErrType(err, NoNode) { + log.Warningf("GetShard(%v, %v) returned ErrNoNode, consider checking the topology.", keyspace, shard) + } else { + vterrors.Wrapf(err, "GetShard(%v, %v) failed", keyspace, shard) } - mu.Lock() - result[shard] = si - mu.Unlock() - }(shard) - } - wg.Wait() - if rec.HasErrors() { - return nil, rec.Error() + } + result[shard] = si } return result, nil } diff --git a/go/vt/topo/memorytopo/file.go b/go/vt/topo/memorytopo/file.go index f7dda922e48..ddeba947e97 100644 --- a/go/vt/topo/memorytopo/file.go +++ b/go/vt/topo/memorytopo/file.go @@ -168,7 +168,7 @@ func (c *Conn) Delete(ctx context.Context, filePath string, version topo.Version // Check if it's a directory. if n.isDirectory() { //lint:ignore ST1005 Delete is a function name - return fmt.Errorf("Delete(%v, %v) failed: it's a directory", c.cell, filePath) + return fmt.Errorf("delete(%v, %v) failed: it's a directory", c.cell, filePath) } // Check the version. diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 49d75847cf4..30fd5db1cda 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -279,7 +279,7 @@ func (ts *Server) CreateShard(ctx context.Context, keyspace, shard string) (err defer unlock(&err) // validate parameters - name, keyRange, err := ValidateShardName(shard) + _, keyRange, err := ValidateShardName(shard) if err != nil { return err } @@ -288,27 +288,20 @@ func (ts *Server) CreateShard(ctx context.Context, keyspace, shard string) (err KeyRange: keyRange, } - isMasterServing := true - - // start the shard IsMasterServing. If it overlaps with - // other shards for some serving types, remove them. - - if IsShardUsingRangeBasedSharding(name) { - // if we are using range-based sharding, we don't want - // overlapping shards to all serve and confuse the clients. - sis, err := ts.FindAllShardsInKeyspace(ctx, keyspace) - if err != nil && !IsErrType(err, NoNode) { - return err - } - for _, si := range sis { - if si.KeyRange == nil || key.KeyRangesIntersect(si.KeyRange, keyRange) { - isMasterServing = false - } + // Set master as serving only if its keyrange doesn't overlap + // with other shards. This applies to unsharded keyspaces also + value.IsMasterServing = true + sis, err := ts.FindAllShardsInKeyspace(ctx, keyspace) + if err != nil && !IsErrType(err, NoNode) { + return err + } + for _, si := range sis { + if si.KeyRange == nil || key.KeyRangesIntersect(si.KeyRange, keyRange) { + value.IsMasterServing = false + break } } - value.IsMasterServing = isMasterServing - // Marshal and save. data, err := proto.Marshal(value) if err != nil { diff --git a/go/vt/topo/stats_conn_test.go b/go/vt/topo/stats_conn_test.go index 5148be26f48..501268092ea 100644 --- a/go/vt/topo/stats_conn_test.go +++ b/go/vt/topo/stats_conn_test.go @@ -67,7 +67,7 @@ func (st *fakeConn) Get(ctx context.Context, filePath string) (bytes []byte, ver // Delete is part of the Conn interface func (st *fakeConn) Delete(ctx context.Context, filePath string, version Version) (err error) { if filePath == "error" { - return fmt.Errorf("Dummy error") + return fmt.Errorf("dummy error") } return err } @@ -75,7 +75,7 @@ func (st *fakeConn) Delete(ctx context.Context, filePath string, version Version // Lock is part of the Conn interface func (st *fakeConn) Lock(ctx context.Context, dirPath, contents string) (lock LockDescriptor, err error) { if dirPath == "error" { - return lock, fmt.Errorf("Dummy error") + return lock, fmt.Errorf("dummy error") } return lock, err @@ -89,7 +89,7 @@ func (st *fakeConn) Watch(ctx context.Context, filePath string) (current *WatchD // NewMasterParticipation is part of the Conn interface func (st *fakeConn) NewMasterParticipation(name, id string) (mp MasterParticipation, err error) { if name == "error" { - return mp, fmt.Errorf("Dummy error") + return mp, fmt.Errorf("dummy error") } return mp, err diff --git a/go/vt/topotools/shard_test.go b/go/vt/topotools/shard_test.go index 521d163d378..68ab7be9a99 100644 --- a/go/vt/topotools/shard_test.go +++ b/go/vt/topotools/shard_test.go @@ -55,9 +55,11 @@ func TestCreateShard(t *testing.T) { } } -// TestCreateShardCustomSharding checks ServedTypes is set correctly -// when creating multiple custom sharding shards -func TestCreateShardCustomSharding(t *testing.T) { +// TestCreateShardMultiUnsharded checks ServedTypes is set +// only for the first created shard. +// TODO(sougou): we should eventually disallow multiple shards +// for unsharded keyspaces. +func TestCreateShardMultiUnsharded(t *testing.T) { ctx := context.Background() // Set up topology. @@ -90,7 +92,7 @@ func TestCreateShardCustomSharding(t *testing.T) { if si, err := ts.GetShard(ctx, keyspace, shard1); err != nil { t.Fatalf("GetShard(shard1) failed: %v", err) } else { - if !si.IsMasterServing { + if si.IsMasterServing { t.Fatalf("shard1 should have all 3 served types") } } diff --git a/go/vt/topotools/split.go b/go/vt/topotools/split.go index 98c93e0eb6b..04431ddd699 100644 --- a/go/vt/topotools/split.go +++ b/go/vt/topotools/split.go @@ -17,14 +17,66 @@ limitations under the License. package topotools import ( + "errors" "fmt" "sort" "golang.org/x/net/context" "vitess.io/vitess/go/vt/key" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" ) +// ValidateForReshard returns an error if sourceShards cannot reshard into +// targetShards. +func ValidateForReshard(sourceShards, targetShards []*topo.ShardInfo) error { + for _, source := range sourceShards { + for _, target := range targetShards { + if key.KeyRangeEqual(source.KeyRange, target.KeyRange) { + return fmt.Errorf("same keyrange is present in source and target: %v", key.KeyRangeString(source.KeyRange)) + } + } + } + sourcekr, err := combineKeyRanges(sourceShards) + if err != nil { + return err + } + targetkr, err := combineKeyRanges(targetShards) + if err != nil { + return err + } + if !key.KeyRangeEqual(sourcekr, targetkr) { + return fmt.Errorf("source and target keyranges don't match: %v vs %v", key.KeyRangeString(sourcekr), key.KeyRangeString(targetkr)) + } + return nil +} + +func combineKeyRanges(shards []*topo.ShardInfo) (*topodatapb.KeyRange, error) { + if len(shards) == 0 { + return nil, fmt.Errorf("there are no shards to combine") + } + result := shards[0].KeyRange + krmap := make(map[string]*topodatapb.KeyRange) + for _, si := range shards[1:] { + krmap[si.ShardName()] = si.KeyRange + } + for len(krmap) != 0 { + foundOne := false + for k, kr := range krmap { + newkr, ok := key.KeyRangeAdd(result, kr) + if ok { + foundOne = true + result = newkr + delete(krmap, k) + } + } + if !foundOne { + return nil, errors.New("shards don't form a contiguous keyrange") + } + } + return result, nil +} + // OverlappingShards contains sets of shards that overlap which each-other. // With this library, there is no guarantee of which set will be left or right. type OverlappingShards struct { diff --git a/go/vt/topotools/split_test.go b/go/vt/topotools/split_test.go index 7ee34a15946..5dbbc0f686a 100644 --- a/go/vt/topotools/split_test.go +++ b/go/vt/topotools/split_test.go @@ -20,6 +20,7 @@ import ( "encoding/hex" "testing" + "github.com/stretchr/testify/assert" "vitess.io/vitess/go/vt/topo" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -94,6 +95,64 @@ func compareResultLists(t *testing.T, os []*OverlappingShards, expected []expect } } +func TestValidateForReshard(t *testing.T) { + testcases := []struct { + sources []string + targets []string + out string + }{{ + sources: []string{"-80", "80-"}, + targets: []string{"-40", "40-"}, + out: "", + }, { + sources: []string{"80-", "-80"}, + targets: []string{"-40", "40-"}, + out: "", + }, { + sources: []string{"-40", "40-80", "80-"}, + targets: []string{"-30", "30-"}, + out: "", + }, { + sources: []string{"0"}, + targets: []string{"-40", "40-"}, + out: "", + }, { + sources: []string{"-40", "40-80", "80-"}, + targets: []string{"-40", "40-"}, + out: "same keyrange is present in source and target: -40", + }, { + sources: []string{"-30", "30-80"}, + targets: []string{"-40", "40-"}, + out: "source and target keyranges don't match: -80 vs -", + }, { + sources: []string{"-30", "20-80"}, + targets: []string{"-40", "40-"}, + out: "shards don't form a contiguous keyrange", + }} + buildShards := func(shards []string) []*topo.ShardInfo { + sis := make([]*topo.ShardInfo, 0, len(shards)) + for _, shard := range shards { + _, kr, err := topo.ValidateShardName(shard) + if err != nil { + panic(err) + } + sis = append(sis, topo.NewShardInfo("", shard, &topodatapb.Shard{KeyRange: kr}, nil)) + } + return sis + } + + for _, tcase := range testcases { + sources := buildShards(tcase.sources) + targets := buildShards(tcase.targets) + err := ValidateForReshard(sources, targets) + if tcase.out == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tcase.out) + } + } +} + func TestFindOverlappingShardsNoOverlap(t *testing.T) { var shardMap map[string]*topo.ShardInfo var os []*OverlappingShards diff --git a/go/vt/vtctl/query.go b/go/vt/vtctl/query.go index b6f068fedc5..3a97e961aaf 100644 --- a/go/vt/vtctl/query.go +++ b/go/vt/vtctl/query.go @@ -196,7 +196,7 @@ func commandVtGateExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags * qr, err := session.Execute(ctx, subFlags.Arg(0), bindVars) if err != nil { //lint:ignore ST1005 function name - return fmt.Errorf("Execute failed: %v", err) + return fmt.Errorf("execute failed: %v", err) } if *json { return printJSON(wr.Logger(), qr) @@ -445,7 +445,7 @@ func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags }, subFlags.Arg(1), bindVars, int64(*transactionID), executeOptions) if err != nil { //lint:ignore ST1005 function name - return fmt.Errorf("Execute failed: %v", err) + return fmt.Errorf("execute failed: %v", err) } if *json { return printJSON(wr.Logger(), qr) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index d8cd3d18e75..a7050ded940 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -307,6 +307,9 @@ var commands = []commandGroup{ {"ValidateKeyspace", commandValidateKeyspace, "[-ping-tablets] ", "Validates that all nodes reachable from the specified keyspace are consistent."}, + {"Reshard", commandReshard, + "[-skip_schema_copy] ", + "Start a Resharding process. Example: Reshard ks.workflow001 '0' '-80,80-'"}, {"SplitClone", commandSplitClone, " ", "Start the SplitClone process to perform horizontal resharding. Example: SplitClone ks '0' '-80,80-'"}, @@ -314,7 +317,7 @@ var commands = []commandGroup{ " ", "Start the VerticalSplitClone process to perform vertical resharding. Example: SplitClone from_ks to_ks 'a,/b.*/'"}, {"VDiff", commandVDiff, - "-workflow= [-source_cell=] [-target_cell=] [-tablet_types=REPLICA] [-filtered_replication_wait_time=30s]", + "[-source_cell=] [-target_cell=] [-tablet_types=replica] [-filtered_replication_wait_time=30s] ", "Perform a diff of all tables in the workflow"}, {"MigrateServedTypes", commandMigrateServedTypes, "[-cells=c1,c2,...] [-reverse] [-skip-refresh-state] ", @@ -323,10 +326,10 @@ var commands = []commandGroup{ "[-cells=c1,c2,...] [-reverse] ", "Makes the serve the given type. This command also rebuilds the serving graph."}, {"MigrateReads", commandMigrateReads, - "[-cells=c1,c2,...] [-reverse] -workflow=workflow ", + "[-cells=c1,c2,...] [-reverse] -tablet_type={replica|rdonly} ", "Migrate read traffic for the specified workflow."}, {"MigrateWrites", commandMigrateWrites, - "[-filtered_replication_wait_time=30s] [-cancel] [-reverse_replication=false] -workflow=workflow ", + "[-filtered_replication_wait_time=30s] [-cancel] [-reverse_replication=false] ", "Migrate write traffic for the specified workflow."}, {"CancelResharding", commandCancelResharding, "", @@ -1784,6 +1787,23 @@ func commandValidateKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlag return wr.ValidateKeyspace(ctx, keyspace, *pingTablets) } +func commandReshard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + skipSchemaCopy := subFlags.Bool("skip_schema_copy", false, "Skip copying of schema to targets") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 3 { + return fmt.Errorf("three arguments are required: , source_shards, target_shards") + } + keyspace, workflow, err := splitKeyspaceWorkflow(subFlags.Arg(0)) + if err != nil { + return err + } + source := strings.Split(subFlags.Arg(1), ",") + target := strings.Split(subFlags.Arg(2), ",") + return wr.Reshard(ctx, keyspace, workflow, source, target, *skipSchemaCopy) +} + func commandSplitClone(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { if err := subFlags.Parse(args); err != nil { return err @@ -1811,7 +1831,6 @@ func commandVerticalSplitClone(ctx context.Context, wr *wrangler.Wrangler, subFl } func commandVDiff(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { - workflow := subFlags.String("workflow", "", "Specifies the workflow name") sourceCell := subFlags.String("source_cell", "", "The source cell to compare from") targetCell := subFlags.String("target_cell", "", "The target cell to compare with") tabletTypes := subFlags.String("tablet_types", "", "Tablet types for source and target") @@ -1819,16 +1838,28 @@ func commandVDiff(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.Fla if err := subFlags.Parse(args); err != nil { return err } + if subFlags.NArg() != 1 { - return fmt.Errorf("the is required") + return fmt.Errorf(" is required") + } + keyspace, workflow, err := splitKeyspaceWorkflow(subFlags.Arg(0)) + if err != nil { + return err } - targetKeyspace := subFlags.Arg(0) - _, err := wr.VDiff(ctx, targetKeyspace, *workflow, *sourceCell, *targetCell, *tabletTypes, *filteredReplicationWaitTime, + _, err = wr.VDiff(ctx, keyspace, workflow, *sourceCell, *targetCell, *tabletTypes, *filteredReplicationWaitTime, *HealthCheckTopologyRefresh, *HealthcheckRetryDelay, *HealthCheckTimeout) return err } +func splitKeyspaceWorkflow(in string) (keyspace, workflow string, err error) { + splits := strings.Split(in, ".") + if len(splits) != 2 { + return "", "", fmt.Errorf("invalid format for : %s", in) + } + return splits[0], splits[1], nil +} + func commandMigrateServedTypes(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { cellsStr := subFlags.String("cells", "", "Specifies a comma-separated list of cells to update") reverse := subFlags.Bool("reverse", false, "Moves the served tablet type backward instead of forward.") @@ -1889,16 +1920,15 @@ func commandMigrateServedFrom(ctx context.Context, wr *wrangler.Wrangler, subFla func commandMigrateReads(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { reverse := subFlags.Bool("reverse", false, "Moves the served tablet type backward instead of forward.") cellsStr := subFlags.String("cells", "", "Specifies a comma-separated list of cells to update") - workflow := subFlags.String("workflow", "", "Specifies the workflow name") + tabletType := subFlags.String("tablet_type", "", "Tablet type (replica or rdonly)") if err := subFlags.Parse(args); err != nil { return err } - if subFlags.NArg() != 2 { - return fmt.Errorf("the and arguments are required for the MigrateReads command") - } - keyspace := subFlags.Arg(0) - servedType, err := parseTabletType(subFlags.Arg(2), []topodatapb.TabletType{topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY}) + if *tabletType == "" { + return fmt.Errorf("-tablet_type must be specified") + } + servedType, err := parseTabletType(*tabletType, []topodatapb.TabletType{topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY}) if err != nil { return err } @@ -1910,29 +1940,34 @@ func commandMigrateReads(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if *reverse { direction = wrangler.DirectionBackward } - if *workflow == "" { - return fmt.Errorf("a -workflow=workflow argument is required") + if subFlags.NArg() != 1 { + return fmt.Errorf(" is required") + } + keyspace, workflow, err := splitKeyspaceWorkflow(subFlags.Arg(0)) + if err != nil { + return err } - return wr.MigrateReads(ctx, keyspace, *workflow, servedType, cells, direction) + + return wr.MigrateReads(ctx, keyspace, workflow, servedType, cells, direction) } func commandMigrateWrites(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { filteredReplicationWaitTime := subFlags.Duration("filtered_replication_wait_time", 30*time.Second, "Specifies the maximum time to wait, in seconds, for filtered replication to catch up on master migrations. The migration will be aborted on timeout.") reverseReplication := subFlags.Bool("reverse_replication", true, "Also reverse the replication") cancelMigrate := subFlags.Bool("cancel", false, "Cancel the failed migration and serve from source") - workflow := subFlags.String("workflow", "", "Specifies the workflow name") if err := subFlags.Parse(args); err != nil { return err } + if subFlags.NArg() != 1 { - return fmt.Errorf("the argument is required for the MigrateWrites command") + return fmt.Errorf(" is required") } - - keyspace := subFlags.Arg(0) - if *workflow == "" { - return fmt.Errorf("a -workflow=workflow argument is required") + keyspace, workflow, err := splitKeyspaceWorkflow(subFlags.Arg(0)) + if err != nil { + return err } - journalID, err := wr.MigrateWrites(ctx, keyspace, *workflow, *filteredReplicationWaitTime, *cancelMigrate, *reverseReplication) + + journalID, err := wr.MigrateWrites(ctx, keyspace, workflow, *filteredReplicationWaitTime, *cancelMigrate, *reverseReplication) if err != nil { return err } diff --git a/go/vt/vtgate/endtoend/deletetest/delete_test.go b/go/vt/vtgate/endtoend/deletetest/delete_test.go index 9f0cb6ceb79..de6614ed7c9 100644 --- a/go/vt/vtgate/endtoend/deletetest/delete_test.go +++ b/go/vt/vtgate/endtoend/deletetest/delete_test.go @@ -127,7 +127,7 @@ func TestMain(m *testing.M) { }}, }}, } - cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")} + cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")} if err := cfg.InitSchemas("ks", schema, vschema); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.RemoveAll(cfg.SchemaDir) diff --git a/go/vt/vtgate/endtoend/main_test.go b/go/vt/vtgate/endtoend/main_test.go index da52636dcec..a3b69cdb6ea 100644 --- a/go/vt/vtgate/endtoend/main_test.go +++ b/go/vt/vtgate/endtoend/main_test.go @@ -171,7 +171,7 @@ func TestMain(m *testing.M) { }}, }}, } - cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")} + cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")} if err := cfg.InitSchemas("ks", schema, vschema); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.RemoveAll(cfg.SchemaDir) diff --git a/go/vt/vtgate/endtoend/vstream_test.go b/go/vt/vtgate/endtoend/vstream_test.go index 1d382bc0f00..f38e0c47e07 100644 --- a/go/vt/vtgate/endtoend/vstream_test.go +++ b/go/vt/vtgate/endtoend/vstream_test.go @@ -88,6 +88,7 @@ func TestVStream(t *testing.T) { if err != nil { t.Fatal(err) } + fmt.Printf("events: %v\n", events) // An empty transaction has three events: begin, gtid and commit. if len(events) == 3 && !emptyEventSkipped { emptyEventSkipped = true @@ -107,7 +108,7 @@ func TestVStream(t *testing.T) { Type: querypb.Type_INT64, }}, } - gotFields := events[2].FieldEvent + gotFields := events[1].FieldEvent if !proto.Equal(gotFields, wantFields) { t.Errorf("FieldEvent:\n%v, want\n%v", gotFields, wantFields) } @@ -120,7 +121,7 @@ func TestVStream(t *testing.T) { }, }}, } - gotRows := events[3].RowEvent + gotRows := events[2].RowEvent if !proto.Equal(gotRows, wantRows) { t.Errorf("RowEvent:\n%v, want\n%v", gotRows, wantRows) } diff --git a/go/vt/vtgate/engine/delete.go b/go/vt/vtgate/engine/delete.go index 9d59c69181e..22e2fd61a5c 100644 --- a/go/vt/vtgate/engine/delete.go +++ b/go/vt/vtgate/engine/delete.go @@ -50,7 +50,7 @@ type Delete struct { Query string // Vindex specifies the vindex to be used. - Vindex vindexes.Vindex + Vindex vindexes.SingleColumn // Values specifies the vindex values to use for routing. // For now, only one value is specified. Values []sqltypes.PlanValue diff --git a/go/vt/vtgate/engine/delete_test.go b/go/vt/vtgate/engine/delete_test.go index a61dcc702e7..d2b7c9173af 100644 --- a/go/vt/vtgate/engine/delete_test.go +++ b/go/vt/vtgate/engine/delete_test.go @@ -65,7 +65,7 @@ func TestDeleteEqual(t *testing.T) { Sharded: true, }, Query: "dummy_delete", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -98,7 +98,7 @@ func TestDeleteEqualNoRoute(t *testing.T) { Sharded: true, }, Query: "dummy_delete", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -127,7 +127,7 @@ func TestDeleteEqualNoScatter(t *testing.T) { Sharded: true, }, Query: "dummy_delete", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -142,7 +142,7 @@ func TestDeleteOwnedVindex(t *testing.T) { Opcode: DeleteEqual, Keyspace: ks.Keyspace, Query: "dummy_delete", - Vindex: ks.Vindexes["hash"], + Vindex: ks.Vindexes["hash"].(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, Table: ks.Tables["t1"], OwnedVindexQuery: "dummy_subquery", diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index bf1de16defb..bf75e6a4468 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -391,19 +391,10 @@ func (ins *Insert) getInsertShardedRoute(vcursor VCursor, bindVars map[string]*q // keyspace ids. For regular inserts, a failure to find a route // results in an error. For 'ignore' type inserts, the keyspace // id is returned as nil, which is used later to drop the corresponding rows. - colVindex := ins.Table.ColumnVindexes[0] - keyspaceIDs, err := ins.processPrimary(vcursor, vindexRowsValues[0], colVindex) + keyspaceIDs, err := ins.processPrimary(vcursor, vindexRowsValues[0], ins.Table.ColumnVindexes[0]) if err != nil { return nil, nil, vterrors.Wrap(err, "getInsertShardedRoute") } - // Primary vindex can be owned. If so, go through the processOwned flow. - // If not owned, we don't do processUnowned because there's no need to verify - // the keyspace ids we just generated. - if colVindex.Owned { - if err := ins.processOwned(vcursor, vindexRowsValues[0], colVindex, keyspaceIDs); err != nil { - return nil, nil, vterrors.Wrap(err, "getInsertShardedRoute") - } - } for vIdx := 1; vIdx < len(ins.Table.ColumnVindexes); vIdx++ { colVindex := ins.Table.ColumnVindexes[vIdx] diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index d8373863f93..7989ae64f3b 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -20,7 +20,6 @@ import ( "errors" "testing" - "github.com/stretchr/testify/assert" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -655,9 +654,14 @@ func TestInsertShardedGeo(t *testing.T) { Type: "region_experimental", Params: map[string]string{ "region_bytes": "1", - "table": "lkp", - "from": "id,region", - "to": "toc", + }, + }, + "lookup": { + Type: "lookup_unique", + Params: map[string]string{ + "table": "id_idx", + "from": "id", + "to": "keyspace_id", }, Owner: "t1", }, @@ -666,7 +670,10 @@ func TestInsertShardedGeo(t *testing.T) { "t1": { ColumnVindexes: []*vschemapb.ColumnVindex{{ Name: "geo", - Columns: []string{"id", "region"}, + Columns: []string{"region", "id"}, + }, { + Name: "lookup", + Columns: []string{"id"}, }}, }, }, @@ -683,20 +690,30 @@ func TestInsertShardedGeo(t *testing.T) { InsertSharded, ks.Keyspace, []sqltypes.PlanValue{{ - // colVindex columns: id, region + // colVindex columns: region, id Values: []sqltypes.PlanValue{{ + // rows for region + Values: []sqltypes.PlanValue{{ + Value: sqltypes.NewInt64(1), + }, { + Value: sqltypes.NewInt64(255), + }}, + }, { // rows for id Values: []sqltypes.PlanValue{{ Value: sqltypes.NewInt64(1), }, { Value: sqltypes.NewInt64(1), }}, - }, { - // rows for region + }}, + }, { + // colVindex columns: id + Values: []sqltypes.PlanValue{{ + // rows for id Values: []sqltypes.PlanValue{{ Value: sqltypes.NewInt64(1), }, { - Value: sqltypes.NewInt64(255), + Value: sqltypes.NewInt64(1), }}, }}, }}, @@ -715,11 +732,9 @@ func TestInsertShardedGeo(t *testing.T) { t.Fatal(err) } vc.ExpectLog(t, []string{ - // ExecutePre proves that keyspace ids are generated, and that they are inserted into the lookup. - `ExecutePre insert into lkp(id, region, toc) values(:id0, :region0, :toc0), (:id1, :region1, :toc1) ` + + `Execute insert into id_idx(id, keyspace_id) values(:id0, :keyspace_id0), (:id1, :keyspace_id1) ` + `id0: type:INT64 value:"1" id1: type:INT64 value:"1" ` + - `region0: type:INT64 value:"1" region1: type:INT64 value:"255" ` + - `toc0: type:VARBINARY value:"\001\026k@\264J\272K\326" toc1: type:VARBINARY value:"\377\026k@\264J\272K\326" true`, + `keyspace_id0: type:VARBINARY value:"\001\026k@\264J\272K\326" keyspace_id1: type:VARBINARY value:"\377\026k@\264J\272K\326" true`, `ResolveDestinations sharded [value:"0" value:"1" ] Destinations:DestinationKeyspaceID(01166b40b44aba4bd6),DestinationKeyspaceID(ff166b40b44aba4bd6)`, `ExecuteMultiShard sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:01166b40b44aba4bd6 */ ` + `{_id0: type:INT64 value:"1" _id1: type:INT64 value:"1" ` + @@ -922,104 +937,6 @@ func TestInsertShardedIgnoreOwned(t *testing.T) { }) } -func TestInsertIgnoreGeo(t *testing.T) { - invschema := &vschemapb.SrvVSchema{ - Keyspaces: map[string]*vschemapb.Keyspace{ - "sharded": { - Sharded: true, - Vindexes: map[string]*vschemapb.Vindex{ - "geo": { - Type: "region_experimental", - Params: map[string]string{ - "region_bytes": "1", - "table": "lkp", - "from": "id,region", - "to": "toc", - }, - Owner: "t1", - }, - }, - Tables: map[string]*vschemapb.Table{ - "t1": { - ColumnVindexes: []*vschemapb.ColumnVindex{{ - Name: "geo", - Columns: []string{"id", "region"}, - }}, - }, - }, - }, - }, - } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } - ks := vs.Keyspaces["sharded"] - - ins := NewInsert( - InsertShardedIgnore, - ks.Keyspace, - []sqltypes.PlanValue{{ - // colVindex columns: id, region - Values: []sqltypes.PlanValue{{ - // rows for id - Values: []sqltypes.PlanValue{{ - Value: sqltypes.NewInt64(1), - }, { - Value: sqltypes.NewInt64(2), - }}, - }, { - // rows for region - Values: []sqltypes.PlanValue{{ - Value: sqltypes.NewInt64(1), - }, { - Value: sqltypes.NewInt64(2), - }}, - }}, - }}, - ks.Tables["t1"], - "prefix", - []string{" mid1", " mid2"}, - " suffix", - ) - - ksid0 := sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "to", - "varbinary", - ), - "\x00", - ) - noresult := &sqltypes.Result{} - vc := &loggingVCursor{ - shards: []string{"-20", "20-"}, - shardForKsid: []string{"20-", "-20"}, - results: []*sqltypes.Result{ - // insert lkp - noresult, - // fail one verification (row 2) - ksid0, - noresult, - }, - } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } - vc.ExpectLog(t, []string{ - `ExecutePre insert ignore into lkp(id, region, toc) values(:id0, :region0, :toc0), (:id1, :region1, :toc1) ` + - `id0: type:INT64 value:"1" id1: type:INT64 value:"2" ` + - `region0: type:INT64 value:"1" region1: type:INT64 value:"2" ` + - `toc0: type:VARBINARY value:"\001\026k@\264J\272K\326" toc1: type:VARBINARY value:"\002\006\347\352\"\316\222p\217" true`, - // Row 2 will fail verification. This is what we're testing. The second row should not get inserted. - `ExecutePre select id from lkp where id = :id and toc = :toc id: type:INT64 value:"1" toc: type:VARBINARY value:"\001\026k@\264J\272K\326" false`, - `ExecutePre select id from lkp where id = :id and toc = :toc id: type:INT64 value:"2" toc: type:VARBINARY value:"\002\006\347\352\"\316\222p\217" false`, - `ResolveDestinations sharded [value:"0" ] Destinations:DestinationKeyspaceID(01166b40b44aba4bd6)`, - `ExecuteMultiShard sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:01166b40b44aba4bd6 */ ` + - `{_id0: type:INT64 value:"1" _region0: type:INT64 value:"1" } true true`, - }) -} - func TestInsertShardedIgnoreOwnedWithNull(t *testing.T) { invschema := &vschemapb.SrvVSchema{ Keyspaces: map[string]*vschemapb.Keyspace{ @@ -1270,91 +1187,6 @@ func TestInsertShardedUnownedVerify(t *testing.T) { }) } -func TestInsertUnownedGeo(t *testing.T) { - invschema := &vschemapb.SrvVSchema{ - Keyspaces: map[string]*vschemapb.Keyspace{ - "sharded": { - Sharded: true, - Vindexes: map[string]*vschemapb.Vindex{ - "primary": { - Type: "hash", - }, - "geo": { - Type: "region_experimental", - Params: map[string]string{ - "region_bytes": "1", - "table": "lkp", - "from": "other_id,region", - "to": "toc", - }, - }, - }, - Tables: map[string]*vschemapb.Table{ - "t1": { - ColumnVindexes: []*vschemapb.ColumnVindex{{ - Name: "primary", - Columns: []string{"id"}, - }, { - Name: "geo", - Columns: []string{"other_id", "region"}, - }}, - }, - }, - }, - }, - } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } - ks := vs.Keyspaces["sharded"] - - ins := NewInsert( - InsertSharded, - ks.Keyspace, - []sqltypes.PlanValue{{ - // colVindex columns: id - Values: []sqltypes.PlanValue{{ - // rows for id - Values: []sqltypes.PlanValue{{ - Value: sqltypes.NewInt64(1), - }}, - }}, - }, { - // colVindex columns: other_id, region - Values: []sqltypes.PlanValue{{ - // rows for other_id - Values: []sqltypes.PlanValue{{ - Value: sqltypes.NewInt64(2), - }}, - }, { - // rows for region - Values: []sqltypes.PlanValue{{ - Value: sqltypes.NewInt64(3), - }}, - }}, - }}, - ks.Tables["t1"], - "prefix", - []string{" mid1"}, - " suffix", - ) - - noresult := &sqltypes.Result{} - vc := &loggingVCursor{ - shards: []string{"-20", "20-"}, - results: []*sqltypes.Result{ - // fail verification - noresult, - }, - } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) - assert.EqualError(t, err, "execInsertSharded: getInsertShardedRoute: values [[INT64(2) INT64(3)]] for column [other_id region] does not map to keyspace ids") - vc.ExpectLog(t, []string{ - `ExecutePre select other_id from lkp where other_id = :other_id and toc = :toc other_id: type:INT64 value:"2" toc: type:VARBINARY value:"\026k@\264J\272K\326" false`, - }) -} - func TestInsertShardedIgnoreUnownedVerify(t *testing.T) { invschema := &vschemapb.SrvVSchema{ Keyspaces: map[string]*vschemapb.Keyspace{ diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 2ad95038820..4576894f13c 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -65,7 +65,7 @@ type Route struct { FieldQuery string // Vindex specifies the vindex to be used. - Vindex vindexes.Vindex + Vindex vindexes.SingleColumn // Values specifies the vindex values to use for routing. Values []sqltypes.PlanValue @@ -463,7 +463,7 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { return out, err } -func resolveSingleShard(vcursor VCursor, vindex vindexes.Vindex, keyspace *vindexes.Keyspace, vindexKey sqltypes.Value) (*srvtopo.ResolvedShard, []byte, error) { +func resolveSingleShard(vcursor VCursor, vindex vindexes.SingleColumn, keyspace *vindexes.Keyspace, vindexKey sqltypes.Value) (*srvtopo.ResolvedShard, []byte, error) { destinations, err := vindex.Map(vcursor, []sqltypes.Value{vindexKey}) if err != nil { return nil, nil, err diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index 5e34bb5f4ce..5855e2e5873 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -119,7 +119,7 @@ func TestSelectEqualUnique(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}} vc := &loggingVCursor{ @@ -164,7 +164,7 @@ func TestSelectEqualUniqueScatter(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}} vc := &loggingVCursor{ @@ -208,7 +208,7 @@ func TestSelectEqual(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}} vc := &loggingVCursor{ @@ -264,7 +264,7 @@ func TestSelectEqualNoRoute(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}} vc := &loggingVCursor{shards: []string{"-20", "20-"}} @@ -301,7 +301,7 @@ func TestSelectINUnique(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{ Values: []sqltypes.PlanValue{{ Value: sqltypes.NewInt64(1), @@ -357,7 +357,7 @@ func TestSelectINNonUnique(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{ Values: []sqltypes.PlanValue{{ Value: sqltypes.NewInt64(1), @@ -542,7 +542,7 @@ func TestRouteGetFields(t *testing.T) { "dummy_select", "dummy_select_field", ) - sel.Vindex = vindex + sel.Vindex = vindex.(vindexes.SingleColumn) sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}} vc := &loggingVCursor{shards: []string{"-20", "20-"}} diff --git a/go/vt/vtgate/engine/update.go b/go/vt/vtgate/engine/update.go index 075a5a8c93c..dc3c5915468 100644 --- a/go/vt/vtgate/engine/update.go +++ b/go/vt/vtgate/engine/update.go @@ -50,7 +50,7 @@ type Update struct { Query string // Vindex specifies the vindex to be used. - Vindex vindexes.Vindex + Vindex vindexes.SingleColumn // Values specifies the vindex values to use for routing. // For now, only one value is specified. Values []sqltypes.PlanValue diff --git a/go/vt/vtgate/engine/update_test.go b/go/vt/vtgate/engine/update_test.go index fcd51942ffa..d21f62990f7 100644 --- a/go/vt/vtgate/engine/update_test.go +++ b/go/vt/vtgate/engine/update_test.go @@ -66,7 +66,7 @@ func TestUpdateEqual(t *testing.T) { Sharded: true, }, Query: "dummy_update", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -95,7 +95,7 @@ func TestUpdateScatter(t *testing.T) { Sharded: true, }, Query: "dummy_update", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -118,7 +118,7 @@ func TestUpdateScatter(t *testing.T) { Sharded: true, }, Query: "dummy_update", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, MultiShardAutocommit: true, } @@ -148,7 +148,7 @@ func TestUpdateEqualNoRoute(t *testing.T) { Sharded: true, }, Query: "dummy_update", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -177,7 +177,7 @@ func TestUpdateEqualNoScatter(t *testing.T) { Sharded: true, }, Query: "dummy_update", - Vindex: vindex, + Vindex: vindex.(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, } @@ -192,7 +192,7 @@ func TestUpdateEqualChangedVindex(t *testing.T) { Opcode: UpdateEqual, Keyspace: ks.Keyspace, Query: "dummy_update", - Vindex: ks.Vindexes["hash"], + Vindex: ks.Vindexes["hash"].(vindexes.SingleColumn), Values: []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}, ChangedVindexValues: map[string][]sqltypes.PlanValue{ "twocol": {{ diff --git a/go/vt/vtgate/engine/vindex_func.go b/go/vt/vtgate/engine/vindex_func.go index 99c0a153195..8ffb5914055 100644 --- a/go/vt/vtgate/engine/vindex_func.go +++ b/go/vt/vtgate/engine/vindex_func.go @@ -35,8 +35,9 @@ type VindexFunc struct { // Fields is the field info for the result. Fields []*querypb.Field // Cols contains source column numbers: 0 for id, 1 for keyspace_id. - Cols []int - Vindex vindexes.Vindex + Cols []int + // TODO(sougou): add support for MultiColumn. + Vindex vindexes.SingleColumn Value sqltypes.PlanValue } diff --git a/go/vt/vtgate/engine/vindex_func_test.go b/go/vt/vtgate/engine/vindex_func_test.go index 917fc73f1b8..338ca9f7ddd 100644 --- a/go/vt/vtgate/engine/vindex_func_test.go +++ b/go/vt/vtgate/engine/vindex_func_test.go @@ -246,7 +246,7 @@ func TestFieldOrder(t *testing.T) { } } -func testVindexFunc(v vindexes.Vindex) *VindexFunc { +func testVindexFunc(v vindexes.SingleColumn) *VindexFunc { return &VindexFunc{ Fields: sqltypes.MakeTestFields("id|keyspace_id|range_start|range_end", "varbinary|varbinary|varbinary|varbinary"), Cols: []int{0, 1, 2, 3}, diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 51440ed5308..b3c9e2c8980 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -1280,7 +1280,11 @@ func (e *Executor) MessageAck(ctx context.Context, keyspace, name string, ids [] } // We always use the (unique) primary vindex. The ID must be the // primary vindex for message tables. - destinations, err := table.ColumnVindexes[0].Vindex.Map(vcursor, values) + single, ok := table.ColumnVindexes[0].Vindex.(vindexes.SingleColumn) + if !ok { + return 0, fmt.Errorf("multi-column vindexes not supported") + } + destinations, err := single.Map(vcursor, values) if err != nil { return 0, err } diff --git a/go/vt/vtgate/logstats.go b/go/vt/vtgate/logstats.go index 2e9f8a3588b..47f34e2d7ef 100644 --- a/go/vt/vtgate/logstats.go +++ b/go/vt/vtgate/logstats.go @@ -27,8 +27,10 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/streamlog" + "vitess.io/vitess/go/tb" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/callinfo" + "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" ) @@ -125,6 +127,14 @@ func (stats *LogStats) Logf(w io.Writer, params url.Values) error { return nil } + // FormatBindVariables call might panic so we're going to catch it here + // and print out the stack trace for debugging. + defer func() { + if x := recover(); x != nil { + log.Errorf("Uncaught panic:\n%v\n%s", x, tb.Stack(4)) + } + }() + formattedBindVars := "\"[REDACTED]\"" if !*streamlog.RedactDebugUIQueries { _, fullBindParams := params["full"] diff --git a/go/vt/vtgate/planbuilder/from.go b/go/vt/vtgate/planbuilder/from.go index 7f919ca21c0..2c2813bce10 100644 --- a/go/vt/vtgate/planbuilder/from.go +++ b/go/vt/vtgate/planbuilder/from.go @@ -195,7 +195,11 @@ func (pb *primitiveBuilder) buildTablePrimitive(tableExpr *sqlparser.AliasedTabl return err } if vindex != nil { - pb.bldr, pb.st = newVindexFunc(alias, vindex) + single, ok := vindex.(vindexes.SingleColumn) + if !ok { + return fmt.Errorf("multi-column vindexes not supported") + } + pb.bldr, pb.st = newVindexFunc(alias, single) return nil } @@ -244,7 +248,8 @@ func (pb *primitiveBuilder) buildTablePrimitive(tableExpr *sqlparser.AliasedTabl // Use the Binary vindex, which is the identity function // for keyspace id. eroute = engine.NewSimpleRoute(engine.SelectEqualUnique, vst.Keyspace) - eroute.Vindex, _ = vindexes.NewBinary("binary", nil) + vindex, _ = vindexes.NewBinary("binary", nil) + eroute.Vindex, _ = vindex.(vindexes.SingleColumn) eroute.Values = []sqltypes.PlanValue{{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, vst.Pinned)}} } // set table name into route diff --git a/go/vt/vtgate/planbuilder/route_option.go b/go/vt/vtgate/planbuilder/route_option.go index 5a4f45f60a7..1d6ba1c3679 100644 --- a/go/vt/vtgate/planbuilder/route_option.go +++ b/go/vt/vtgate/planbuilder/route_option.go @@ -37,7 +37,7 @@ type routeOption struct { // vindexMap is a map of all vindexMap that can be used // for the routeOption. - vindexMap map[*column]vindexes.Vindex + vindexMap map[*column]vindexes.SingleColumn // condition stores the AST condition that will be used // to resolve the ERoute Values field. @@ -58,7 +58,7 @@ func newSimpleRouteOption(rb *route, eroute *engine.Route) *routeOption { } } -func newRouteOption(rb *route, vst *vindexes.Table, sub *tableSubstitution, vindexMap map[*column]vindexes.Vindex, eroute *engine.Route) *routeOption { +func newRouteOption(rb *route, vst *vindexes.Table, sub *tableSubstitution, vindexMap map[*column]vindexes.SingleColumn, eroute *engine.Route) *routeOption { var subs []*tableSubstitution if sub != nil && sub.newExpr != nil { subs = []*tableSubstitution{sub} @@ -95,7 +95,7 @@ func (ro *routeOption) MergeJoin(rro *routeOption, isLeftJoin bool) { // Add RHS vindexes only if it's not a left join. for c, v := range rro.vindexMap { if ro.vindexMap == nil { - ro.vindexMap = make(map[*column]vindexes.Vindex) + ro.vindexMap = make(map[*column]vindexes.SingleColumn) } ro.vindexMap[c] = v } @@ -126,7 +126,7 @@ func (ro *routeOption) MergeUnion(rro *routeOption) { ro.substitutions = append(ro.substitutions, rro.substitutions...) } -func (ro *routeOption) SubqueryToTable(rb *route, vindexMap map[*column]vindexes.Vindex) { +func (ro *routeOption) SubqueryToTable(rb *route, vindexMap map[*column]vindexes.SingleColumn) { ro.rb = rb ro.vschemaTable = nil ro.vindexMap = vindexMap @@ -234,14 +234,14 @@ func (ro *routeOption) UpdatePlan(pb *primitiveBuilder, filter sqlparser.Expr) { } } -func (ro *routeOption) updateRoute(opcode engine.RouteOpcode, vindex vindexes.Vindex, condition sqlparser.Expr) { +func (ro *routeOption) updateRoute(opcode engine.RouteOpcode, vindex vindexes.SingleColumn, condition sqlparser.Expr) { ro.eroute.Opcode = opcode ro.eroute.Vindex = vindex ro.condition = condition } // computePlan computes the plan for the specified filter. -func (ro *routeOption) computePlan(pb *primitiveBuilder, filter sqlparser.Expr) (opcode engine.RouteOpcode, vindex vindexes.Vindex, condition sqlparser.Expr) { +func (ro *routeOption) computePlan(pb *primitiveBuilder, filter sqlparser.Expr) (opcode engine.RouteOpcode, vindex vindexes.SingleColumn, condition sqlparser.Expr) { switch node := filter.(type) { case *sqlparser.ComparisonExpr: switch node.Operator { @@ -257,7 +257,7 @@ func (ro *routeOption) computePlan(pb *primitiveBuilder, filter sqlparser.Expr) } // computeEqualPlan computes the plan for an equality constraint. -func (ro *routeOption) computeEqualPlan(pb *primitiveBuilder, comparison *sqlparser.ComparisonExpr) (opcode engine.RouteOpcode, vindex vindexes.Vindex, condition sqlparser.Expr) { +func (ro *routeOption) computeEqualPlan(pb *primitiveBuilder, comparison *sqlparser.ComparisonExpr) (opcode engine.RouteOpcode, vindex vindexes.SingleColumn, condition sqlparser.Expr) { left := comparison.Left right := comparison.Right vindex = ro.FindVindex(pb, left) @@ -278,7 +278,7 @@ func (ro *routeOption) computeEqualPlan(pb *primitiveBuilder, comparison *sqlpar } // computeINPlan computes the plan for an IN constraint. -func (ro *routeOption) computeINPlan(pb *primitiveBuilder, comparison *sqlparser.ComparisonExpr) (opcode engine.RouteOpcode, vindex vindexes.Vindex, condition sqlparser.Expr) { +func (ro *routeOption) computeINPlan(pb *primitiveBuilder, comparison *sqlparser.ComparisonExpr) (opcode engine.RouteOpcode, vindex vindexes.SingleColumn, condition sqlparser.Expr) { vindex = ro.FindVindex(pb, comparison.Left) if vindex == nil { return engine.SelectScatter, nil, nil @@ -323,7 +323,7 @@ func (ro *routeOption) isBetterThan(other *routeOption) bool { return false } -func (ro *routeOption) FindVindex(pb *primitiveBuilder, expr sqlparser.Expr) vindexes.Vindex { +func (ro *routeOption) FindVindex(pb *primitiveBuilder, expr sqlparser.Expr) vindexes.SingleColumn { col, ok := expr.(*sqlparser.ColName) if !ok { return nil diff --git a/go/vt/vtgate/planbuilder/route_option_test.go b/go/vt/vtgate/planbuilder/route_option_test.go index 425aa3d34f5..c39dfe48b02 100644 --- a/go/vt/vtgate/planbuilder/route_option_test.go +++ b/go/vt/vtgate/planbuilder/route_option_test.go @@ -168,10 +168,11 @@ func TestIsBetterThan(t *testing.T) { case 2: v, _ = newLookupIndex("", nil) } + single, _ := v.(vindexes.SingleColumn) return &routeOption{ eroute: &engine.Route{ Opcode: opt, - Vindex: v, + Vindex: single, }, } } diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index fb9ad2301c2..654245c3187 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -87,13 +87,13 @@ func newSymtabWithRoute(rb *route) *symtab { // AddVSchemaTable takes a list of vschema tables as input and // creates a table with multiple route options. It returns a // list of vindex maps, one for each input. -func (st *symtab) AddVSchemaTable(alias sqlparser.TableName, vschemaTables []*vindexes.Table, rb *route) (vindexMaps []map[*column]vindexes.Vindex, err error) { +func (st *symtab) AddVSchemaTable(alias sqlparser.TableName, vschemaTables []*vindexes.Table, rb *route) (vindexMaps []map[*column]vindexes.SingleColumn, err error) { t := &table{ alias: alias, origin: rb, } - vindexMaps = make([]map[*column]vindexes.Vindex, len(vschemaTables)) + vindexMaps = make([]map[*column]vindexes.SingleColumn, len(vschemaTables)) for i, vst := range vschemaTables { // The following logic allows the first table to be authoritative while the rest // are not. But there's no need to reveal this flexibility to the user. @@ -115,8 +115,12 @@ func (st *symtab) AddVSchemaTable(alias sqlparser.TableName, vschemaTables []*vi t.isAuthoritative = true } - var vindexMap map[*column]vindexes.Vindex + var vindexMap map[*column]vindexes.SingleColumn for _, cv := range vst.ColumnVindexes { + single, ok := cv.Vindex.(vindexes.SingleColumn) + if !ok { + continue + } for j, cvcol := range cv.Columns { col, err := t.mergeColumn(cvcol, &column{ origin: rb, @@ -128,10 +132,10 @@ func (st *symtab) AddVSchemaTable(alias sqlparser.TableName, vschemaTables []*vi if j == 0 { // For now, only the first column is used for vindex Map functions. if vindexMap == nil { - vindexMap = make(map[*column]vindexes.Vindex) + vindexMap = make(map[*column]vindexes.SingleColumn) } - if vindexMap[col] == nil || vindexMap[col].Cost() > cv.Vindex.Cost() { - vindexMap[col] = cv.Vindex + if vindexMap[col] == nil || vindexMap[col].Cost() > single.Cost() { + vindexMap[col] = single } } } diff --git a/go/vt/vtgate/planbuilder/symtab_test.go b/go/vt/vtgate/planbuilder/symtab_test.go index 34006ff4b2e..81e85e889b7 100644 --- a/go/vt/vtgate/planbuilder/symtab_test.go +++ b/go/vt/vtgate/planbuilder/symtab_test.go @@ -28,6 +28,8 @@ func TestSymtabAddVSchemaTable(t *testing.T) { tname := sqlparser.TableName{Name: sqlparser.NewTableIdent("t")} rb := &route{} + null, _ := vindexes.CreateVindex("null", "null", nil) + tcases := []struct { in []*vindexes.Table authoritative bool @@ -49,6 +51,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { in: []*vindexes.Table{{ ColumnVindexes: []*vindexes.ColumnVindex{{ Columns: []sqlparser.ColIdent{sqlparser.NewColIdent("C1")}, + Vindex: null, }}, Columns: []vindexes.Column{{ Name: sqlparser.NewColIdent("C1"), @@ -66,6 +69,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { sqlparser.NewColIdent("C1"), sqlparser.NewColIdent("C2"), }, + Vindex: null, }}, Columns: []vindexes.Column{{ Name: sqlparser.NewColIdent("C1"), @@ -94,6 +98,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { in: []*vindexes.Table{{ ColumnVindexes: []*vindexes.ColumnVindex{{ Columns: []sqlparser.ColIdent{sqlparser.NewColIdent("C1")}, + Vindex: null, }}, Columns: []vindexes.Column{{ Name: sqlparser.NewColIdent("C2"), @@ -109,6 +114,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { sqlparser.NewColIdent("C1"), sqlparser.NewColIdent("C2"), }, + Vindex: null, }}, }}, authoritative: false, @@ -145,12 +151,14 @@ func TestSymtabAddVSchemaTable(t *testing.T) { Columns: []sqlparser.ColIdent{ sqlparser.NewColIdent("C1"), }, + Vindex: null, }}, }, { ColumnVindexes: []*vindexes.ColumnVindex{{ Columns: []sqlparser.ColIdent{ sqlparser.NewColIdent("C2"), }, + Vindex: null, }}, }}, authoritative: false, @@ -162,10 +170,12 @@ func TestSymtabAddVSchemaTable(t *testing.T) { Columns: []sqlparser.ColIdent{ sqlparser.NewColIdent("C1"), }, + Vindex: null, }, { Columns: []sqlparser.ColIdent{ sqlparser.NewColIdent("C2"), }, + Vindex: null, }}, }}, authoritative: false, @@ -246,6 +256,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { Columns: []sqlparser.ColIdent{ sqlparser.NewColIdent("C2"), }, + Vindex: null, }}, }}, err: "column C2 not found in t", diff --git a/go/vt/vtgate/planbuilder/update.go b/go/vt/vtgate/planbuilder/update.go index 78a8f7a14ae..f8bd77f7937 100644 --- a/go/vt/vtgate/planbuilder/update.go +++ b/go/vt/vtgate/planbuilder/update.go @@ -204,7 +204,7 @@ func generateQuery(statement sqlparser.Statement) string { // getDMLRouting returns the vindex and values for the DML, // If it cannot find a unique vindex match, it returns an error. -func getDMLRouting(where *sqlparser.Where, table *vindexes.Table) (vindexes.Vindex, []sqltypes.PlanValue, error) { +func getDMLRouting(where *sqlparser.Where, table *vindexes.Table) (vindexes.SingleColumn, []sqltypes.PlanValue, error) { if where == nil { return nil, nil, errors.New("unsupported: multi-shard where clause in DML") } @@ -212,8 +212,12 @@ func getDMLRouting(where *sqlparser.Where, table *vindexes.Table) (vindexes.Vind if !index.Vindex.IsUnique() { continue } + single, ok := index.Vindex.(vindexes.SingleColumn) + if !ok { + continue + } if pv, ok := getMatch(where.Expr, index.Columns[0]); ok { - return index.Vindex, []sqltypes.PlanValue{pv}, nil + return single, []sqltypes.PlanValue{pv}, nil } } return nil, nil, errors.New("unsupported: multi-shard where clause in DML") diff --git a/go/vt/vtgate/planbuilder/vindex_func.go b/go/vt/vtgate/planbuilder/vindex_func.go index 6135338c120..3902f4a1030 100644 --- a/go/vt/vtgate/planbuilder/vindex_func.go +++ b/go/vt/vtgate/planbuilder/vindex_func.go @@ -40,7 +40,7 @@ type vindexFunc struct { eVindexFunc *engine.VindexFunc } -func newVindexFunc(alias sqlparser.TableName, vindex vindexes.Vindex) (*vindexFunc, *symtab) { +func newVindexFunc(alias sqlparser.TableName, vindex vindexes.SingleColumn) (*vindexFunc, *symtab) { vf := &vindexFunc{ order: 1, eVindexFunc: &engine.VindexFunc{ diff --git a/go/vt/vtgate/vindexes/binary.go b/go/vt/vtgate/vindexes/binary.go index 1d9e2bf132e..50d0500a9bd 100644 --- a/go/vt/vtgate/vindexes/binary.go +++ b/go/vt/vtgate/vindexes/binary.go @@ -25,8 +25,8 @@ import ( ) var ( - _ Vindex = (*Binary)(nil) - _ Reversible = (*Binary)(nil) + _ SingleColumn = (*Binary)(nil) + _ Reversible = (*Binary)(nil) ) // Binary is a vindex that converts binary bits to a keyspace id. diff --git a/go/vt/vtgate/vindexes/binary_test.go b/go/vt/vtgate/vindexes/binary_test.go index a6fe555558a..ac9d5666097 100644 --- a/go/vt/vtgate/vindexes/binary_test.go +++ b/go/vt/vtgate/vindexes/binary_test.go @@ -26,10 +26,11 @@ import ( "vitess.io/vitess/go/vt/key" ) -var binOnlyVindex Vindex +var binOnlyVindex SingleColumn func init() { - binOnlyVindex, _ = CreateVindex("binary", "binary_varchar", nil) + vindex, _ := CreateVindex("binary", "binary_varchar", nil) + binOnlyVindex = vindex.(SingleColumn) } func TestBinaryCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/binarymd5.go b/go/vt/vtgate/vindexes/binarymd5.go index a75326e276e..be6cea79311 100644 --- a/go/vt/vtgate/vindexes/binarymd5.go +++ b/go/vt/vtgate/vindexes/binarymd5.go @@ -25,7 +25,7 @@ import ( ) var ( - _ Vindex = (*BinaryMD5)(nil) + _ SingleColumn = (*BinaryMD5)(nil) ) // BinaryMD5 is a vindex that hashes binary bits to a keyspace id. diff --git a/go/vt/vtgate/vindexes/binarymd5_test.go b/go/vt/vtgate/vindexes/binarymd5_test.go index e3a133c1fac..f959fb4cd19 100644 --- a/go/vt/vtgate/vindexes/binarymd5_test.go +++ b/go/vt/vtgate/vindexes/binarymd5_test.go @@ -26,10 +26,11 @@ import ( "vitess.io/vitess/go/vt/key" ) -var binVindex Vindex +var binVindex SingleColumn func init() { - binVindex, _ = CreateVindex("binary_md5", "binary_md5_varchar", nil) + vindex, _ := CreateVindex("binary_md5", "binary_md5_varchar", nil) + binVindex = vindex.(SingleColumn) } func TestBinaryMD5Cost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/consistent_lookup.go b/go/vt/vtgate/vindexes/consistent_lookup.go index 546b70c244e..6d862e7c8ce 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup.go +++ b/go/vt/vtgate/vindexes/consistent_lookup.go @@ -33,10 +33,10 @@ import ( ) var ( - _ Vindex = (*ConsistentLookupUnique)(nil) + _ SingleColumn = (*ConsistentLookupUnique)(nil) _ Lookup = (*ConsistentLookupUnique)(nil) _ WantOwnerInfo = (*ConsistentLookupUnique)(nil) - _ Vindex = (*ConsistentLookup)(nil) + _ SingleColumn = (*ConsistentLookup)(nil) _ Lookup = (*ConsistentLookup)(nil) _ WantOwnerInfo = (*ConsistentLookup)(nil) ) diff --git a/go/vt/vtgate/vindexes/consistent_lookup_test.go b/go/vt/vtgate/vindexes/consistent_lookup_test.go index e1e571d07ec..bb25c204273 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup_test.go +++ b/go/vt/vtgate/vindexes/consistent_lookup_test.go @@ -412,7 +412,7 @@ func TestConsistentLookupNoUpdate(t *testing.T) { vc.verifyLog(t, []string{}) } -func createConsistentLookup(t *testing.T, name string) Vindex { +func createConsistentLookup(t *testing.T, name string) SingleColumn { t.Helper() l, err := CreateVindex(name, name, map[string]string{ "table": "t", @@ -429,7 +429,7 @@ func createConsistentLookup(t *testing.T, name string) Vindex { if err := l.(WantOwnerInfo).SetOwnerInfo("ks", "t1", cols); err != nil { t.Fatal(err) } - return l + return l.(SingleColumn) } type loggingVCursor struct { diff --git a/go/vt/vtgate/vindexes/hash.go b/go/vt/vtgate/vindexes/hash.go index 43859e4f38a..1811750402f 100644 --- a/go/vt/vtgate/vindexes/hash.go +++ b/go/vt/vtgate/vindexes/hash.go @@ -31,13 +31,15 @@ import ( ) var ( - _ Vindex = (*Hash)(nil) - _ Reversible = (*Hash)(nil) + _ SingleColumn = (*Hash)(nil) + _ Reversible = (*Hash)(nil) ) // Hash defines vindex that hashes an int64 to a KeyspaceId -// by using null-key 3DES hash. It's Unique, Reversible and +// by using null-key DES hash. It's Unique, Reversible and // Functional. +// Note that at once stage we used a 3DES-based hash here, +// but for a null key as in our case, they are completely equivalent. type Hash struct { name string } @@ -114,11 +116,11 @@ func (vind *Hash) ReverseMap(_ VCursor, ksids [][]byte) ([]sqltypes.Value, error return reverseIds, nil } -var block3DES cipher.Block +var blockDES cipher.Block func init() { var err error - block3DES, err = des.NewTripleDESCipher(make([]byte, 24)) + blockDES, err = des.NewCipher(make([]byte, 8)) if err != nil { panic(err) } @@ -128,7 +130,7 @@ func init() { func vhash(shardKey uint64) []byte { var keybytes, hashed [8]byte binary.BigEndian.PutUint64(keybytes[:], shardKey) - block3DES.Encrypt(hashed[:], keybytes[:]) + blockDES.Encrypt(hashed[:], keybytes[:]) return []byte(hashed[:]) } @@ -137,6 +139,6 @@ func vunhash(k []byte) (uint64, error) { return 0, fmt.Errorf("invalid keyspace id: %v", hex.EncodeToString(k)) } var unhashed [8]byte - block3DES.Decrypt(unhashed[:], k) + blockDES.Decrypt(unhashed[:], k) return binary.BigEndian.Uint64(unhashed[:]), nil } diff --git a/go/vt/vtgate/vindexes/hash_test.go b/go/vt/vtgate/vindexes/hash_test.go index c0e811a3616..847cf1968f0 100644 --- a/go/vt/vtgate/vindexes/hash_test.go +++ b/go/vt/vtgate/vindexes/hash_test.go @@ -25,14 +25,14 @@ import ( "vitess.io/vitess/go/vt/key" ) -var hash Vindex +var hash SingleColumn func init() { hv, err := CreateVindex("hash", "nn", map[string]string{"Table": "t", "Column": "c"}) if err != nil { panic(err) } - hash = hv + hash = hv.(SingleColumn) } func TestHashCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index a6869066e4c..6f13abb0e56 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -27,10 +27,10 @@ import ( ) var ( - _ Vindex = (*LookupUnique)(nil) - _ Lookup = (*LookupUnique)(nil) - _ Vindex = (*LookupNonUnique)(nil) - _ Lookup = (*LookupNonUnique)(nil) + _ SingleColumn = (*LookupUnique)(nil) + _ Lookup = (*LookupUnique)(nil) + _ SingleColumn = (*LookupNonUnique)(nil) + _ Lookup = (*LookupNonUnique)(nil) ) func init() { diff --git a/go/vt/vtgate/vindexes/lookup_hash.go b/go/vt/vtgate/vindexes/lookup_hash.go index 30118bd274a..46809ad4b78 100644 --- a/go/vt/vtgate/vindexes/lookup_hash.go +++ b/go/vt/vtgate/vindexes/lookup_hash.go @@ -27,10 +27,10 @@ import ( ) var ( - _ Vindex = (*LookupHash)(nil) - _ Lookup = (*LookupHash)(nil) - _ Vindex = (*LookupHashUnique)(nil) - _ Lookup = (*LookupHashUnique)(nil) + _ SingleColumn = (*LookupHash)(nil) + _ Lookup = (*LookupHash)(nil) + _ SingleColumn = (*LookupHashUnique)(nil) + _ Lookup = (*LookupHashUnique)(nil) ) func init() { diff --git a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go index bd67e77523d..27ea2cb4e2d 100644 --- a/go/vt/vtgate/vindexes/lookup_hash_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_hash_unique_test.go @@ -31,12 +31,13 @@ func TestLookupHashUniqueNew(t *testing.T) { t.Errorf("Create(lookup, false): %v, want %v", got, want) } - l, _ = CreateVindex("lookup_hash_unique", "lookup_hash_unique", map[string]string{ + vindex, _ := CreateVindex("lookup_hash_unique", "lookup_hash_unique", map[string]string{ "table": "t", "from": "fromc", "to": "toc", "write_only": "true", }) + l = vindex.(SingleColumn) if want, got := l.(*LookupHashUnique).writeOnly, true; got != want { t.Errorf("Create(lookup, false): %v, want %v", got, want) } diff --git a/go/vt/vtgate/vindexes/lookup_test.go b/go/vt/vtgate/vindexes/lookup_test.go index ca26ce61e9d..12865b8fb75 100644 --- a/go/vt/vtgate/vindexes/lookup_test.go +++ b/go/vt/vtgate/vindexes/lookup_test.go @@ -185,7 +185,7 @@ func TestLookupNonUniqueMap(t *testing.T) { } func TestLookupNonUniqueMapAutocommit(t *testing.T) { - lookupNonUnique, err := CreateVindex("lookup", "lookup", map[string]string{ + vindex, err := CreateVindex("lookup", "lookup", map[string]string{ "table": "t", "from": "fromc", "to": "toc", @@ -194,6 +194,7 @@ func TestLookupNonUniqueMapAutocommit(t *testing.T) { if err != nil { t.Fatal(err) } + lookupNonUnique := vindex.(SingleColumn) vc := &vcursor{numRows: 2} got, err := lookupNonUnique.Map(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}) @@ -325,7 +326,7 @@ func TestLookupNonUniqueVerify(t *testing.T) { } func TestLookupNonUniqueVerifyAutocommit(t *testing.T) { - lookupNonUnique, err := CreateVindex("lookup", "lookup", map[string]string{ + vindex, err := CreateVindex("lookup", "lookup", map[string]string{ "table": "t", "from": "fromc", "to": "toc", @@ -334,6 +335,7 @@ func TestLookupNonUniqueVerifyAutocommit(t *testing.T) { if err != nil { t.Fatal(err) } + lookupNonUnique := vindex.(SingleColumn) vc := &vcursor{numRows: 1} _, err = lookupNonUnique.Verify(vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}, [][]byte{[]byte("test1"), []byte("test2")}) @@ -549,7 +551,7 @@ func TestLookupNonUniqueUpdate(t *testing.T) { } } -func createLookup(t *testing.T, name string, writeOnly bool) Vindex { +func createLookup(t *testing.T, name string, writeOnly bool) SingleColumn { t.Helper() write := "false" if writeOnly { @@ -564,5 +566,5 @@ func createLookup(t *testing.T, name string, writeOnly bool) Vindex { if err != nil { t.Fatal(err) } - return l + return l.(SingleColumn) } diff --git a/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash.go b/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash.go index 30b12a33f30..f56d302c819 100644 --- a/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash.go +++ b/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash.go @@ -28,10 +28,10 @@ import ( ) var ( - _ Vindex = (*LookupUnicodeLooseMD5Hash)(nil) - _ Lookup = (*LookupUnicodeLooseMD5Hash)(nil) - _ Vindex = (*LookupUnicodeLooseMD5HashUnique)(nil) - _ Lookup = (*LookupUnicodeLooseMD5HashUnique)(nil) + _ SingleColumn = (*LookupUnicodeLooseMD5Hash)(nil) + _ Lookup = (*LookupUnicodeLooseMD5Hash)(nil) + _ SingleColumn = (*LookupUnicodeLooseMD5HashUnique)(nil) + _ Lookup = (*LookupUnicodeLooseMD5HashUnique)(nil) ) func init() { diff --git a/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash_test.go b/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash_test.go index dc87eaf6aa9..abe193974e7 100644 --- a/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash_test.go +++ b/go/vt/vtgate/vindexes/lookup_unicodeloosemd5_hash_test.go @@ -82,7 +82,7 @@ func TestLookupUnicodeLooseMD5HashMap(t *testing.T) { } func TestLookupUnicodeLooseMD5HashMapAutocommit(t *testing.T) { - lookupNonUnique, err := CreateVindex("lookup_unicodeloosemd5_hash", "lookup", map[string]string{ + vindex, err := CreateVindex("lookup_unicodeloosemd5_hash", "lookup", map[string]string{ "table": "t", "from": "fromc", "to": "toc", @@ -92,6 +92,7 @@ func TestLookupUnicodeLooseMD5HashMapAutocommit(t *testing.T) { if err != nil { t.Fatal(err) } + lookupNonUnique := vindex.(SingleColumn) vc := &vcursor{numRows: 2} got, err := lookupNonUnique.Map(vc, []sqltypes.Value{sqltypes.NewInt64(10), sqltypes.NewInt64(20)}) @@ -229,7 +230,7 @@ func TestLookupUnicodeLooseMD5HashVerify(t *testing.T) { } func TestLookupUnicodeLooseMD5HashVerifyAutocommit(t *testing.T) { - lookupNonUnique, err := CreateVindex("lookup_unicodeloosemd5_hash", "lookup", map[string]string{ + vindex, err := CreateVindex("lookup_unicodeloosemd5_hash", "lookup", map[string]string{ "table": "t", "from": "fromc", "to": "toc", @@ -238,6 +239,7 @@ func TestLookupUnicodeLooseMD5HashVerifyAutocommit(t *testing.T) { if err != nil { t.Fatal(err) } + lookupNonUnique := vindex.(SingleColumn) vc := &vcursor{numRows: 1} _, err = lookupNonUnique.Verify(vc, []sqltypes.Value{sqltypes.NewInt64(10), sqltypes.NewInt64(20)}, diff --git a/go/vt/vtgate/vindexes/lookup_unique_test.go b/go/vt/vtgate/vindexes/lookup_unique_test.go index 5e33bf370a9..cb9ab2bed8b 100644 --- a/go/vt/vtgate/vindexes/lookup_unique_test.go +++ b/go/vt/vtgate/vindexes/lookup_unique_test.go @@ -33,12 +33,13 @@ func TestLookupUniqueNew(t *testing.T) { t.Errorf("Create(lookup, false): %v, want %v", got, want) } - l, _ = CreateVindex("lookup_unique", "lookup_unique", map[string]string{ + vindex, _ := CreateVindex("lookup_unique", "lookup_unique", map[string]string{ "table": "t", "from": "fromc", "to": "toc", "write_only": "true", }) + l = vindex.(SingleColumn) if want, got := l.(*LookupUnique).writeOnly, true; got != want { t.Errorf("Create(lookup, false): %v, want %v", got, want) } diff --git a/go/vt/vtgate/vindexes/null_test.go b/go/vt/vtgate/vindexes/null_test.go index f92855edca5..11568f9eb5c 100644 --- a/go/vt/vtgate/vindexes/null_test.go +++ b/go/vt/vtgate/vindexes/null_test.go @@ -25,14 +25,14 @@ import ( "vitess.io/vitess/go/vt/key" ) -var null Vindex +var null SingleColumn func init() { hv, err := CreateVindex("null", "nn", map[string]string{"Table": "t", "Column": "c"}) if err != nil { panic(err) } - null = hv + null = hv.(SingleColumn) } func TestNullCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/numeric.go b/go/vt/vtgate/vindexes/numeric.go index 1b0b5c2a071..8ebdbe5c2d6 100644 --- a/go/vt/vtgate/vindexes/numeric.go +++ b/go/vt/vtgate/vindexes/numeric.go @@ -27,8 +27,8 @@ import ( ) var ( - _ Vindex = (*Numeric)(nil) - _ Reversible = (*Numeric)(nil) + _ SingleColumn = (*Numeric)(nil) + _ Reversible = (*Numeric)(nil) ) // Numeric defines a bit-pattern mapping of a uint64 to the KeyspaceId. diff --git a/go/vt/vtgate/vindexes/numeric_static_map.go b/go/vt/vtgate/vindexes/numeric_static_map.go index 9f51a80a888..39832aa3777 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map.go +++ b/go/vt/vtgate/vindexes/numeric_static_map.go @@ -30,7 +30,7 @@ import ( ) var ( - _ Vindex = (*NumericStaticMap)(nil) + _ SingleColumn = (*NumericStaticMap)(nil) ) // NumericLookupTable stores the mapping of keys. diff --git a/go/vt/vtgate/vindexes/numeric_static_map_test.go b/go/vt/vtgate/vindexes/numeric_static_map_test.go index e81abe331db..94035320f0d 100644 --- a/go/vt/vtgate/vindexes/numeric_static_map_test.go +++ b/go/vt/vtgate/vindexes/numeric_static_map_test.go @@ -28,10 +28,14 @@ import ( // createVindex creates the "numeric_static_map" vindex object which is used by // each test. -func createVindex() (Vindex, error) { +func createVindex() (SingleColumn, error) { m := make(map[string]string) m["json_path"] = "testdata/numeric_static_map_test.json" - return CreateVindex("numeric_static_map", "numericStaticMap", m) + vindex, err := CreateVindex("numeric_static_map", "numericStaticMap", m) + if err != nil { + panic(err) + } + return vindex.(SingleColumn), nil } func TestNumericStaticMapCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/numeric_test.go b/go/vt/vtgate/vindexes/numeric_test.go index 7ad9a357f69..3d0d0532414 100644 --- a/go/vt/vtgate/vindexes/numeric_test.go +++ b/go/vt/vtgate/vindexes/numeric_test.go @@ -26,10 +26,11 @@ import ( "vitess.io/vitess/go/vt/key" ) -var numeric Vindex +var numeric SingleColumn func init() { - numeric, _ = CreateVindex("numeric", "num", nil) + vindex, _ := CreateVindex("numeric", "num", nil) + numeric = vindex.(SingleColumn) } func TestNumericCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/region_experimental.go b/go/vt/vtgate/vindexes/region_experimental.go index 15159517dd9..ba10a10f7dc 100644 --- a/go/vt/vtgate/vindexes/region_experimental.go +++ b/go/vt/vtgate/vindexes/region_experimental.go @@ -26,10 +26,7 @@ import ( ) var ( - _ Vindex = (*RegionExperimental)(nil) - _ Lookup = (*RegionExperimental)(nil) - _ WantOwnerInfo = (*RegionExperimental)(nil) - _ MultiColumn = (*RegionExperimental)(nil) + _ MultiColumn = (*RegionExperimental)(nil) ) func init() { @@ -40,8 +37,8 @@ func init() { // The table is expected to define the id column as unique. It's // Unique and a Lookup. type RegionExperimental struct { + name string regionBytes int - *ConsistentLookupUnique } // NewRegionExperimental creates a RegionExperimental vindex. @@ -61,45 +58,51 @@ func NewRegionExperimental(name string, m map[string]string) (Vindex, error) { default: return nil, fmt.Errorf("region_bits must be 1 or 2: %v", rbs) } - vindex, err := NewConsistentLookupUnique(name, m) - if err != nil { - // Unreachable. - return nil, err - } - cl := vindex.(*ConsistentLookupUnique) - if len(cl.lkp.FromColumns) != 2 { - return nil, fmt.Errorf("two columns are required for region_experimental: %v", cl.lkp.FromColumns) - } return &RegionExperimental{ - regionBytes: rb, - ConsistentLookupUnique: cl, + name: name, + regionBytes: rb, }, nil } -// MapMulti satisfies MultiColumn. -func (ge *RegionExperimental) MapMulti(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) { +// String returns the name of the vindex. +func (ge *RegionExperimental) String() string { + return ge.name +} + +// Cost returns the cost of this index as 1. +func (ge *RegionExperimental) Cost() int { + return 1 +} + +// IsUnique returns true since the Vindex is unique. +func (ge *RegionExperimental) IsUnique() bool { + return true +} + +// Map satisfies MultiColumn. +func (ge *RegionExperimental) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) { destinations := make([]key.Destination, 0, len(rowsColValues)) for _, row := range rowsColValues { if len(row) != 2 { destinations = append(destinations, key.DestinationNone{}) continue } - // Compute hash. - hn, err := sqltypes.ToUint64(row[0]) + // Compute region prefix. + rn, err := sqltypes.ToUint64(row[0]) if err != nil { destinations = append(destinations, key.DestinationNone{}) continue } - h := vhash(hn) + r := make([]byte, 2, 2+8) + binary.BigEndian.PutUint16(r, uint16(rn)) - // Compute region prefix. - rn, err := sqltypes.ToUint64(row[1]) + // Compute hash. + hn, err := sqltypes.ToUint64(row[1]) if err != nil { destinations = append(destinations, key.DestinationNone{}) continue } - r := make([]byte, 2) - binary.BigEndian.PutUint16(r, uint16(rn)) + h := vhash(hn) // Concatenate and add to destinations. if ge.regionBytes == 1 { @@ -111,10 +114,10 @@ func (ge *RegionExperimental) MapMulti(vcursor VCursor, rowsColValues [][]sqltyp return destinations, nil } -// VerifyMulti satisfies MultiColumn. -func (ge *RegionExperimental) VerifyMulti(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) { +// Verify satisfies MultiColumn. +func (ge *RegionExperimental) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) { result := make([]bool, len(rowsColValues)) - destinations, _ := ge.MapMulti(vcursor, rowsColValues) + destinations, _ := ge.Map(vcursor, rowsColValues) for i, dest := range destinations { destksid, ok := dest.(key.DestinationKeyspaceID) if !ok { @@ -122,14 +125,5 @@ func (ge *RegionExperimental) VerifyMulti(vcursor VCursor, rowsColValues [][]sql } result[i] = bytes.Equal([]byte(destksid), ksids[i]) } - // We also need to verify from the lookup. - // TODO(sougou): we should only verify true values from previous result. - lresult, err := Verify(ge.ConsistentLookupUnique, vcursor, rowsColValues, ksids) - if err != nil { - return nil, err - } - for i := range result { - result[i] = result[i] && lresult[i] - } return result, nil } diff --git a/go/vt/vtgate/vindexes/region_experimental_test.go b/go/vt/vtgate/vindexes/region_experimental_test.go index ef85c65b90e..f1162f43613 100644 --- a/go/vt/vtgate/vindexes/region_experimental_test.go +++ b/go/vt/vtgate/vindexes/region_experimental_test.go @@ -25,23 +25,32 @@ import ( "vitess.io/vitess/go/vt/key" ) -func TestRegionExperimentalMapMulti1(t *testing.T) { +func TestRegionExperimentalMisc(t *testing.T) { ge, err := createRegionVindex(t, "region_experimental", "f1,f2", 1) assert.NoError(t, err) - got, err := ge.(MultiColumn).MapMulti(nil, [][]sqltypes.Value{{ + assert.Equal(t, 1, ge.Cost()) + assert.Equal(t, "region_experimental", ge.String()) + assert.True(t, ge.IsUnique()) +} + +func TestRegionExperimentalMap(t *testing.T) { + vindex, err := createRegionVindex(t, "region_experimental", "f1,f2", 1) + assert.NoError(t, err) + ge := vindex.(MultiColumn) + got, err := ge.Map(nil, [][]sqltypes.Value{{ sqltypes.NewInt64(1), sqltypes.NewInt64(1), }, { - sqltypes.NewInt64(1), sqltypes.NewInt64(255), + sqltypes.NewInt64(255), sqltypes.NewInt64(1), }, { - sqltypes.NewInt64(1), sqltypes.NewInt64(256), + sqltypes.NewInt64(256), sqltypes.NewInt64(1), }, { // Invalid length. sqltypes.NewInt64(1), }, { - // Invalid id. + // Invalid region. sqltypes.NewVarBinary("abcd"), sqltypes.NewInt64(256), }, { - // Invalid region. + // Invalid id. sqltypes.NewInt64(1), sqltypes.NewVarBinary("abcd"), }}) assert.NoError(t, err) @@ -58,16 +67,17 @@ func TestRegionExperimentalMapMulti1(t *testing.T) { } func TestRegionExperimentalMapMulti2(t *testing.T) { - ge, err := createRegionVindex(t, "region_experimental", "f1,f2", 2) + vindex, err := createRegionVindex(t, "region_experimental", "f1,f2", 2) assert.NoError(t, err) - got, err := ge.(MultiColumn).MapMulti(nil, [][]sqltypes.Value{{ + ge := vindex.(MultiColumn) + got, err := ge.Map(nil, [][]sqltypes.Value{{ sqltypes.NewInt64(1), sqltypes.NewInt64(1), }, { - sqltypes.NewInt64(1), sqltypes.NewInt64(255), + sqltypes.NewInt64(255), sqltypes.NewInt64(1), }, { - sqltypes.NewInt64(1), sqltypes.NewInt64(256), + sqltypes.NewInt64(256), sqltypes.NewInt64(1), }, { - sqltypes.NewInt64(1), sqltypes.NewInt64(0x10000), + sqltypes.NewInt64(0x10000), sqltypes.NewInt64(1), }}) assert.NoError(t, err) @@ -81,15 +91,12 @@ func TestRegionExperimentalMapMulti2(t *testing.T) { } func TestRegionExperimentalVerifyMulti(t *testing.T) { - - ge, err := createRegionVindex(t, "region_experimental", "f1,f2", 1) + vindex, err := createRegionVindex(t, "region_experimental", "f1,f2", 1) assert.NoError(t, err) + ge := vindex.(MultiColumn) vals := [][]sqltypes.Value{{ // One for match sqltypes.NewInt64(1), sqltypes.NewInt64(1), - }, { - // One for mismatch by lookup - sqltypes.NewInt64(1), sqltypes.NewInt64(1), }, { // One for mismatch sqltypes.NewInt64(1), sqltypes.NewInt64(1), @@ -98,27 +105,14 @@ func TestRegionExperimentalVerifyMulti(t *testing.T) { sqltypes.NewInt64(1), }} ksids := [][]byte{ - []byte("\x01\x16k@\xb4J\xbaK\xd6"), []byte("\x01\x16k@\xb4J\xbaK\xd6"), []byte("no match"), []byte(""), } - vc := &loggingVCursor{} - vc.AddResult(makeTestResult(1), nil) - // The second value should return a mismatch. - vc.AddResult(&sqltypes.Result{}, nil) - vc.AddResult(makeTestResult(1), nil) - vc.AddResult(makeTestResult(1), nil) - want := []bool{true, false, false, false} - got, err := ge.(MultiColumn).VerifyMulti(vc, vals, ksids) + want := []bool{true, false, false} + got, err := ge.Verify(nil, vals, ksids) assert.NoError(t, err) - vc.verifyLog(t, []string{ - "ExecutePre select f1 from t where f1 = :f1 and toc = :toc [{f1 1} {toc \x01\x16k@\xb4J\xbaK\xd6}] false", - "ExecutePre select f1 from t where f1 = :f1 and toc = :toc [{f1 1} {toc \x01\x16k@\xb4J\xbaK\xd6}] false", - "ExecutePre select f1 from t where f1 = :f1 and toc = :toc [{f1 1} {toc no match}] false", - "ExecutePre select f1 from t where f1 = :f1 and toc = :toc [{f1 1} {toc }] false", - }) assert.Equal(t, want, got) } @@ -127,8 +121,6 @@ func TestRegionExperimentalCreateErrors(t *testing.T) { assert.EqualError(t, err, "region_bits must be 1 or 2: 3") _, err = CreateVindex("region_experimental", "region_experimental", nil) assert.EqualError(t, err, "region_experimental missing region_bytes param") - _, err = createRegionVindex(t, "region_experimental", "f1", 2) - assert.EqualError(t, err, "two columns are required for region_experimental: [f1]") } func createRegionVindex(t *testing.T, name, from string, rb int) (Vindex, error) { diff --git a/go/vt/vtgate/vindexes/reverse_bits.go b/go/vt/vtgate/vindexes/reverse_bits.go index 9957988fe58..f199eaaea35 100644 --- a/go/vt/vtgate/vindexes/reverse_bits.go +++ b/go/vt/vtgate/vindexes/reverse_bits.go @@ -29,8 +29,8 @@ import ( ) var ( - _ Vindex = (*ReverseBits)(nil) - _ Reversible = (*ReverseBits)(nil) + _ SingleColumn = (*ReverseBits)(nil) + _ Reversible = (*ReverseBits)(nil) ) // ReverseBits defines vindex that reverses the bits of a number. diff --git a/go/vt/vtgate/vindexes/reverse_bits_test.go b/go/vt/vtgate/vindexes/reverse_bits_test.go index bfd5a1c8b33..a135b8ca564 100644 --- a/go/vt/vtgate/vindexes/reverse_bits_test.go +++ b/go/vt/vtgate/vindexes/reverse_bits_test.go @@ -25,14 +25,14 @@ import ( "vitess.io/vitess/go/vt/key" ) -var reverseBits Vindex +var reverseBits SingleColumn func init() { hv, err := CreateVindex("reverse_bits", "rr", map[string]string{"Table": "t", "Column": "c"}) if err != nil { panic(err) } - reverseBits = hv + reverseBits = hv.(SingleColumn) } func TestReverseBitsCost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/unicodeloosemd5.go b/go/vt/vtgate/vindexes/unicodeloosemd5.go index 8264ebfe912..ef3a70c9f19 100644 --- a/go/vt/vtgate/vindexes/unicodeloosemd5.go +++ b/go/vt/vtgate/vindexes/unicodeloosemd5.go @@ -30,7 +30,7 @@ import ( ) var ( - _ Vindex = (*UnicodeLooseMD5)(nil) + _ SingleColumn = (*UnicodeLooseMD5)(nil) ) // UnicodeLooseMD5 is a vindex that normalizes and hashes unicode strings diff --git a/go/vt/vtgate/vindexes/unicodeloosemd5_test.go b/go/vt/vtgate/vindexes/unicodeloosemd5_test.go index 9c51f178b1e..89bcc19fbb1 100644 --- a/go/vt/vtgate/vindexes/unicodeloosemd5_test.go +++ b/go/vt/vtgate/vindexes/unicodeloosemd5_test.go @@ -26,10 +26,11 @@ import ( "vitess.io/vitess/go/vt/key" ) -var charVindex Vindex +var charVindex SingleColumn func init() { - charVindex, _ = CreateVindex("unicode_loose_md5", "utf8ch", nil) + vindex, _ := CreateVindex("unicode_loose_md5", "utf8ch", nil) + charVindex = vindex.(SingleColumn) } func TestUnicodeLooseMD5Cost(t *testing.T) { diff --git a/go/vt/vtgate/vindexes/vindex.go b/go/vt/vtgate/vindexes/vindex.go index 35ab8c853e2..3d06668def9 100644 --- a/go/vt/vtgate/vindexes/vindex.go +++ b/go/vt/vtgate/vindexes/vindex.go @@ -22,9 +22,11 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) // This file defines interfaces and registration for vindexes. @@ -56,33 +58,37 @@ type Vindex interface { // IsUnique returns true if the Vindex is unique. // Which means Map() maps to either a KeyRange or a single KeyspaceID. IsUnique() bool +} +// SingleColumn defines the interface for a single column vindex. +type SingleColumn interface { + Vindex // Map can map ids to key.Destination objects. // If the Vindex is unique, each id would map to either // a KeyRange, or a single KeyspaceID. // If the Vindex is non-unique, each id would map to either // a KeyRange, or a list of KeyspaceID. - // If the error returned if nil, then the array len of the - // key.Destination array must match len(ids). Map(vcursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) - // Verify must be implented by all vindexes. It should return - // true if the ids can be mapped to the keyspace ids. + // Verify returns true for every id that successfully maps to the + // specified keyspace id. Verify(vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) } -// MultiColumn defines the interface for vindexes that can -// support multi-column vindexes. +// MultiColumn defines the interface for a multi-column vindex. type MultiColumn interface { - MapMulti(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) - VerifyMulti(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) + Vindex + Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) + Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) } // A Reversible vindex is one that can perform a // reverse lookup from a keyspace id to an id. This // is optional. If present, VTGate can use it to // fill column values based on the target keyspace id. +// Reversible is supported only for SingleColumn vindexes. type Reversible interface { + SingleColumn ReverseMap(vcursor VCursor, ks [][]byte) ([]sqltypes.Value, error) } @@ -140,20 +146,26 @@ func CreateVindex(vindexType, name string, params map[string]string) (Vindex, er return f(name, params) } -// Map invokes MapMulti or Map depending on which is available. +// Map invokes the Map implementation supplied by the vindex. func Map(vindex Vindex, vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) { - if multi, ok := vindex.(MultiColumn); ok { - return multi.MapMulti(vcursor, rowsColValues) + switch vindex := vindex.(type) { + case MultiColumn: + return vindex.Map(vcursor, rowsColValues) + case SingleColumn: + return vindex.Map(vcursor, firstColsOnly(rowsColValues)) } - return vindex.Map(vcursor, firstColsOnly(rowsColValues)) + return nil, vterrors.New(vtrpcpb.Code_INTERNAL, "vindex does not have Map functions") } -// Verify invokes VerifyMulti or Verify depending on which is available. +// Verify invokes the Verify implementation supplied by the vindex. func Verify(vindex Vindex, vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) { - if multi, ok := vindex.(MultiColumn); ok { - return multi.VerifyMulti(vcursor, rowsColValues, ksids) + switch vindex := vindex.(type) { + case MultiColumn: + return vindex.Verify(vcursor, rowsColValues, ksids) + case SingleColumn: + return vindex.Verify(vcursor, firstColsOnly(rowsColValues), ksids) } - return vindex.Verify(vcursor, firstColsOnly(rowsColValues), ksids) + return nil, vterrors.New(vtrpcpb.Code_INTERNAL, "vindex does not have Map functions") } func firstColsOnly(rowsColValues [][]sqltypes.Value) []sqltypes.Value { diff --git a/go/vt/vtgate/vindexes/vindex_test.go b/go/vt/vtgate/vindexes/vindex_test.go index 1c85f83a1fd..c78ab192b26 100644 --- a/go/vt/vtgate/vindexes/vindex_test.go +++ b/go/vt/vtgate/vindexes/vindex_test.go @@ -51,12 +51,10 @@ func TestVindexMap(t *testing.T) { } func TestVindexVerify(t *testing.T) { - vc := &loggingVCursor{} - vc.AddResult(makeTestResult(1), nil) ge, err := createRegionVindex(t, "region_experimental", "f1,f2", 1) assert.NoError(t, err) - got, err := Verify(ge, vc, [][]sqltypes.Value{{ + got, err := Verify(ge, nil, [][]sqltypes.Value{{ sqltypes.NewInt64(1), sqltypes.NewInt64(1), }}, [][]byte{ @@ -64,9 +62,6 @@ func TestVindexVerify(t *testing.T) { }, ) assert.NoError(t, err) - vc.verifyLog(t, []string{ - "ExecutePre select f1 from t where f1 = :f1 and toc = :toc [{f1 1} {toc \x01\x16k@\xb4J\xbaK\xd6}] false", - }) want := []bool{true} assert.Equal(t, want, got) diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index 8f519fb7093..a1f15ad862e 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -313,6 +313,9 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc if !columnVindex.Vindex.IsUnique() { return fmt.Errorf("primary vindex %s is not Unique for table %s", ind.Name, tname) } + if owned { + return fmt.Errorf("primary vindex %s cannot be owned for table %s", ind.Name, tname) + } } t.ColumnVindexes = append(t.ColumnVindexes, columnVindex) if owned { @@ -617,6 +620,10 @@ func FindVindexForSharding(tableName string, colVindexes []*ColumnVindex) (*Colu } result := colVindexes[0] for _, colVindex := range colVindexes { + // Only allow SingleColumn for legacy resharding. + if _, ok := colVindex.Vindex.(SingleColumn); !ok { + continue + } if colVindex.Vindex.Cost() < result.Vindex.Cost() && colVindex.Vindex.IsUnique() { result = colVindex } diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 200636c2388..5ed01c5285c 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -52,7 +52,7 @@ func NewSTFU(name string, params map[string]string) (Vindex, error) { return &stFU{name: name, Params: params}, nil } -var _ Vindex = (*stFU)(nil) +var _ SingleColumn = (*stFU)(nil) // stLN is a Lookup, NonUnique Vindex. type stLN struct { @@ -73,7 +73,7 @@ func NewSTLN(name string, params map[string]string) (Vindex, error) { return &stLN{name: name, Params: params}, nil } -var _ Vindex = (*stLN)(nil) +var _ SingleColumn = (*stLN)(nil) var _ Lookup = (*stLN)(nil) // stLU is a Lookup, Unique Vindex. @@ -95,7 +95,7 @@ func NewSTLU(name string, params map[string]string) (Vindex, error) { return &stLU{name: name, Params: params}, nil } -var _ Vindex = (*stLO)(nil) +var _ SingleColumn = (*stLO)(nil) var _ Lookup = (*stLO)(nil) var _ WantOwnerInfo = (*stLO)(nil) @@ -126,7 +126,7 @@ func NewSTLO(name string, _ map[string]string) (Vindex, error) { return &stLO{name: name}, nil } -var _ Vindex = (*stLO)(nil) +var _ SingleColumn = (*stLO)(nil) var _ Lookup = (*stLO)(nil) func init() { @@ -1554,6 +1554,38 @@ func TestBuildVSchemaNotUniqueFail(t *testing.T) { } } +func TestBuildVSchemaPrimaryCannotBeOwned(t *testing.T) { + bad := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "sharded": { + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "stlu": { + Type: "stlu", + Owner: "t1", + }, + }, + Tables: map[string]*vschemapb.Table{ + "t1": { + ColumnVindexes: []*vschemapb.ColumnVindex{ + { + Column: "c1", + Name: "stlu", + }, + }, + }, + }, + }, + }, + } + got, _ := BuildVSchema(&bad) + err := got.Keyspaces["sharded"].Error + want := "primary vindex stlu cannot be owned for table t1" + if err == nil || err.Error() != want { + t.Errorf("BuildVSchema: %v, want %v", err, want) + } +} + func TestSequence(t *testing.T) { good := vschemapb.SrvVSchema{ Keyspaces: map[string]*vschemapb.Keyspace{ diff --git a/go/vt/vtgate/vindexes/xxhash.go b/go/vt/vtgate/vindexes/xxhash.go new file mode 100644 index 00000000000..fd648e3117d --- /dev/null +++ b/go/vt/vtgate/vindexes/xxhash.go @@ -0,0 +1,88 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "bytes" + "encoding/binary" + + "github.com/cespare/xxhash/v2" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +var ( + _ SingleColumn = (*XXHash)(nil) +) + +// XXHash defines vindex that hashes any sql types to a KeyspaceId +// by using xxhash64. It's Unique and works on any platform giving identical result. +type XXHash struct { + name string +} + +// NewXXHash creates a new XXHash. +func NewXXHash(name string, m map[string]string) (Vindex, error) { + return &XXHash{name: name}, nil +} + +// String returns the name of the vindex. +func (vind *XXHash) String() string { + return vind.name +} + +// Cost returns the cost of this index as 1. +func (vind *XXHash) Cost() int { + return 1 +} + +// IsUnique returns true since the Vindex is unique. +func (vind *XXHash) IsUnique() bool { + return true +} + +// Map can map ids to key.Destination objects. +func (vind *XXHash) Map(cursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) { + out := make([]key.Destination, len(ids)) + for i := range ids { + id := ids[i].ToBytes() + out[i] = key.DestinationKeyspaceID(vXXHash(id)) + } + return out, nil +} + +// Verify returns true if ids maps to ksids. +func (vind *XXHash) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { + out := make([]bool, len(ids)) + for i := range ids { + id := ids[i].ToBytes() + out[i] = bytes.Equal(vXXHash(id), ksids[i]) + } + return out, nil +} + +func init() { + Register("xxhash", NewXXHash) +} + +func vXXHash(shardKey []byte) []byte { + var hashed [8]byte + hashKey := xxhash.Sum64(shardKey) + binary.LittleEndian.PutUint64(hashed[:], hashKey) + return hashed[:] +} diff --git a/go/vt/vtgate/vindexes/xxhash_test.go b/go/vt/vtgate/vindexes/xxhash_test.go new file mode 100644 index 00000000000..187682f9724 --- /dev/null +++ b/go/vt/vtgate/vindexes/xxhash_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/cespare/xxhash/v2" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +var xxHash SingleColumn + +func init() { + hv, err := CreateVindex("xxhash", "xxhash_name", map[string]string{"Table": "t", "Column": "c"}) + if err != nil { + panic(err) + } + xxHash = hv.(SingleColumn) +} + +func TestXXHashCost(t *testing.T) { + if xxHash.Cost() != 1 { + t.Errorf("Cost(): %d, want 1", xxHash.Cost()) + } +} + +func TestXXHashString(t *testing.T) { + if strings.Compare("xxhash_name", xxHash.String()) != 0 { + t.Errorf("String(): %s, want xxhash_name", xxHash.String()) + } +} + +func TestXXHashMap(t *testing.T) { + tcases := []struct { + in sqltypes.Value + out []byte + }{{ + in: sqltypes.NewVarChar("test1"), + out: []byte{0xd0, 0x1a, 0xb7, 0xe4, 0xd6, 0x97, 0x8f, 0xb}, + }, { + in: sqltypes.NewVarChar("test2"), + out: []byte{0x87, 0xeb, 0x11, 0x71, 0x4c, 0xa, 0xe, 0x89}, + }, { + in: sqltypes.NewInt64(1), + out: []byte{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}, + }, { + in: sqltypes.NULL, + out: []byte{0x99, 0xe9, 0xd8, 0x51, 0x37, 0xdb, 0x46, 0xef}, + }, { + in: sqltypes.NewInt64(-1), + out: []byte{0xd8, 0xe2, 0xa6, 0xa7, 0xc8, 0xc7, 0x62, 0x3d}, + }, { + in: sqltypes.NewUint64(18446744073709551615), + out: []byte{0x47, 0x7c, 0xfa, 0x8d, 0x6d, 0x8f, 0x1f, 0x8d}, + }, { + in: sqltypes.NewInt64(9223372036854775807), + out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8}, + }, { + in: sqltypes.NewUint64(9223372036854775807), + out: []byte{0xb3, 0x7e, 0xb0, 0x1f, 0x7b, 0xff, 0xaf, 0xd8}, + }, { + in: sqltypes.NewInt64(-9223372036854775808), + out: []byte{0x10, 0x2c, 0x27, 0xdd, 0xb2, 0x6a, 0x60, 0x9e}, + }} + + for _, tcase := range tcases { + got, err := xxHash.Map(nil, []sqltypes.Value{tcase.in}) + if err != nil { + t.Error(err) + } + out := []byte(got[0].(key.DestinationKeyspaceID)) + if !bytes.Equal(tcase.out, out) { + t.Errorf("Map(%#v): %#v, want %#v", tcase.in, out, tcase.out) + } + } +} + +func TestXXHashVerify(t *testing.T) { + ids := []sqltypes.Value{sqltypes.NewUint64(1), sqltypes.NewUint64(2)} + ksids := [][]byte{{0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}, {0xd4, 0x64, 0x5, 0x36, 0x76, 0x12, 0xb4, 0xb7}} + got, err := xxHash.Verify(nil, ids, ksids) + if err != nil { + t.Fatal(err) + } + want := []bool{true, false} + if !reflect.DeepEqual(got, want) { + t.Errorf("xxHash.Verify: %v, want %v", got, want) + } +} + +func BenchmarkXXHash(b *testing.B) { + for _, benchSize := range []struct { + name string + n int + }{ + {"8B", 8}, + {"64B", 64}, + {"512B", 512}, + {"1KB", 1e3}, + {"4KB", 4e3}, + } { + input := make([]byte, benchSize.n) + for i := range input { + input[i] = byte(i) + } + + name := fmt.Sprintf("xxHash,direct,bytes,n=%s", benchSize.name) + b.Run(name, func(b *testing.B) { + benchmarkHashBytes(b, input) + }) + + } +} + +var sink uint64 + +func benchmarkHashBytes(b *testing.B, input []byte) { + b.SetBytes(int64(len(input))) + for i := 0; i < b.N; i++ { + sink = xxhash.Sum64(input) + } +} diff --git a/go/vt/vttablet/tabletmanager/action_agent.go b/go/vt/vttablet/tabletmanager/action_agent.go index f30ef256128..675176cf2a1 100644 --- a/go/vt/vttablet/tabletmanager/action_agent.go +++ b/go/vt/vttablet/tabletmanager/action_agent.go @@ -308,10 +308,14 @@ func NewActionAgent( return nil, err } + vreplication.InitVStreamerClient(agent.DBConfigs) + // The db name is set by the Start function called above agent.VREngine = vreplication.NewEngine(ts, tabletAlias.Cell, mysqld, func() binlogplayer.DBClient { return binlogplayer.NewDBClient(agent.DBConfigs.FilteredWithDB()) - }, agent.DBConfigs.FilteredWithDB().DbName) + }, + agent.DBConfigs.FilteredWithDB().DbName, + ) servenv.OnTerm(agent.VREngine.Close) // Run a background task to rebuild the SrvKeyspace in our cell/keyspace diff --git a/go/vt/vttablet/tabletmanager/init_tablet.go b/go/vt/vttablet/tabletmanager/init_tablet.go index 24ec839e767..0bf7ae9504f 100644 --- a/go/vt/vttablet/tabletmanager/init_tablet.go +++ b/go/vt/vttablet/tabletmanager/init_tablet.go @@ -40,7 +40,7 @@ import ( ) var ( - initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet") + initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet. Without this flag, the db name defaults to vt_") initKeyspace = flag.String("init_keyspace", "", "(init parameter) keyspace to use for this tablet") initShard = flag.String("init_shard", "", "(init parameter) shard to use for this tablet") initTags flagutil.StringMapValue diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 85c957ecb3f..09d9fc29c3d 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -36,6 +36,7 @@ import ( "vitess.io/vitess/go/vt/topo" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) var ( @@ -49,11 +50,13 @@ var ( // There is no mutex within a controller becaust its members are // either read-only or self-synchronized. type controller struct { + vre *Engine dbClientFactory func() binlogplayer.DBClient mysqld mysqlctl.MysqlDaemon blpStats *binlogplayer.Stats id uint32 + workflow string source binlogdatapb.BinlogSource stopPos string tabletPicker *discovery.TabletPicker @@ -67,11 +70,12 @@ type controller struct { // newController creates a new controller. Unless a stream is explicitly 'Stopped', // this function launches a goroutine to perform continuous vreplication. -func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats) (*controller, error) { +func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats, vre *Engine) (*controller, error) { if blpStats == nil { blpStats = binlogplayer.NewStats() } ct := &controller{ + vre: vre, dbClientFactory: dbClientFactory, mysqld: mysqld, blpStats: blpStats, @@ -84,6 +88,7 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor return nil, err } ct.id = uint32(id) + ct.workflow = params["workflow"] // Nothing to do if replication is stopped. if params["state"] == binlogplayer.BlpStopped { @@ -98,18 +103,20 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor } ct.stopPos = params["stop_pos"] - // tabletPicker - if v, ok := params["cell"]; ok { - cell = v - } - if v, ok := params["tablet_types"]; ok { - tabletTypesStr = v - } - tp, err := discovery.NewTabletPicker(ctx, ts, cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr, *healthcheckTopologyRefresh, *healthcheckRetryDelay, *healthcheckTimeout) - if err != nil { - return nil, err + if ct.source.GetExternalMysql() == "" { + // tabletPicker + if v, ok := params["cell"]; ok { + cell = v + } + if v := params["tablet_types"]; v != "" { + tabletTypesStr = v + } + tp, err := discovery.NewTabletPicker(ctx, ts, cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr, *healthcheckTopologyRefresh, *healthcheckRetryDelay, *healthcheckTimeout) + if err != nil { + return nil, err + } + ct.tabletPicker = tp } - ct.tabletPicker = tp // cancel ctx, ct.cancel = context.WithCancel(ctx) @@ -122,7 +129,9 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor func (ct *controller) run(ctx context.Context) { defer func() { log.Infof("stream %v: stopped", ct.id) - ct.tabletPicker.Close() + if ct.tabletPicker != nil { + ct.tabletPicker.Close() + } close(ct.done) }() @@ -175,11 +184,16 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { } defer dbClient.Close() - tablet, err := ct.tabletPicker.PickForStreaming(ctx) - if err != nil { - return err + var tablet *topodatapb.Tablet + if ct.source.GetExternalMysql() == "" { + log.Infof("trying to find a tablet eligible for vreplication. stream id: %v", ct.id) + tablet, err = ct.tabletPicker.PickForStreaming(ctx) + if err != nil { + return err + } + log.Infof("found a tablet eligible for vreplication. stream id: %v tablet: %s", ct.id, tablet.Alias.String()) + ct.sourceTablet.Set(tablet.Alias.String()) } - ct.sourceTablet.Set(tablet.Alias.String()) switch { case len(ct.source.Tables) > 0: @@ -205,8 +219,16 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { if _, err := dbClient.ExecuteFetch("set names binary", 10000); err != nil { return err } - vreplicator := newVReplicator(ct.id, &ct.source, tablet, ct.blpStats, dbClient, ct.mysqld) - return vreplicator.Replicate(ctx) + + var vsClient VStreamerClient + if ct.source.GetExternalMysql() == "" { + vsClient = NewTabletVStreamerClient(tablet) + } else { + vsClient = NewMySQLVStreamerClient() + } + + vr := newVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld, ct.vre) + return vr.Replicate(ctx) } return fmt.Errorf("missing source") } diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go index b6cf8a3ba11..88d78eb05f7 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go @@ -76,7 +76,7 @@ func TestControllerKeyRange(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) if err != nil { t.Fatal(err) } @@ -136,7 +136,7 @@ func TestControllerTables(t *testing.T) { }, } - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) if err != nil { t.Fatal(err) } @@ -153,7 +153,7 @@ func TestControllerBadID(t *testing.T) { params := map[string]string{ "id": "bad", } - _, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) + _, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil) want := `strconv.Atoi: parsing "bad": invalid syntax` if err == nil || err.Error() != want { t.Errorf("newController err: %v, want %v", err, want) @@ -166,7 +166,7 @@ func TestControllerStopped(t *testing.T) { "state": binlogplayer.BlpStopped, } - ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) + ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestControllerOverrides(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestControllerCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil, nil) if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func TestControllerRetry(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestControllerStopPosition(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) if err != nil { t.Fatal(err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index 3cea952a16e..a9c1678fde3 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -29,6 +29,7 @@ import ( "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/topo" ) @@ -82,6 +83,13 @@ type Engine struct { mysqld mysqlctl.MysqlDaemon dbClientFactory func() binlogplayer.DBClient dbName string + + journaler map[string]*journalEvent +} + +type journalEvent struct { + journal *binlogdatapb.Journal + participants map[string]int } // NewEngine creates a new Engine. @@ -94,6 +102,7 @@ func NewEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, dbClie mysqld: mysqld, dbClientFactory: dbClientFactory, dbName: dbName, + journaler: make(map[string]*journalEvent), } return vre } @@ -187,7 +196,7 @@ func (vre *Engine) initAll() error { return err } for _, row := range rows { - ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil, vre) if err != nil { return err } @@ -280,7 +289,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { if err != nil { return nil, err } - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil, vre) if err != nil { return nil, err } @@ -318,7 +327,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { } // Create a new controller in place of the old one. // For continuity, the new controller inherits the previous stats. - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id]) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id], vre) if err != nil { return nil, err } @@ -394,6 +403,139 @@ func (vre *Engine) fetchIDs(dbClient binlogplayer.DBClient, selector string) (id return ids, bv, nil } +func (vre *Engine) registerJournal(journal *binlogdatapb.Journal, id int) error { + vre.mu.Lock() + defer vre.mu.Unlock() + if !vre.isOpen { + // Unreachable. + return nil + } + + workflow := vre.controllers[id].workflow + key := fmt.Sprintf("%s:%d", workflow, journal.Id) + je, ok := vre.journaler[key] + if !ok { + log.Infof("Journal encountered: %v", journal) + controllerSources := make(map[string]bool) + for _, ct := range vre.controllers { + if ct.workflow != workflow { + // Only compare with streams that belong to the current workflow. + continue + } + ks := fmt.Sprintf("%s:%s", ct.source.Keyspace, ct.source.Shard) + controllerSources[ks] = true + } + je = &journalEvent{ + journal: journal, + participants: make(map[string]int), + } + for _, jks := range journal.Participants { + ks := fmt.Sprintf("%s:%s", jks.Keyspace, jks.Shard) + if _, ok := controllerSources[ks]; !ok { + return fmt.Errorf("cannot redirect on journal: not all sources are present in this workflow: missing %v", ks) + } + je.participants[ks] = 0 + } + vre.journaler[key] = je + } + + ks := fmt.Sprintf("%s:%s", vre.controllers[id].source.Keyspace, vre.controllers[id].source.Shard) + log.Infof("Registering id %v against %v", id, ks) + je.participants[ks] = id + for _, pid := range je.participants { + if pid == 0 { + // Still need to wait. + return nil + } + } + go vre.transitionJournal(key) + return nil +} + +func (vre *Engine) transitionJournal(key string) { + vre.mu.Lock() + defer vre.mu.Unlock() + if !vre.isOpen { + return + } + + log.Infof("Transitioning for journal:workload %v", key) + je := vre.journaler[key] + defer delete(vre.journaler, key) + // Wait for participating controllers to stop. + // Also collect one id reference. + refid := 0 + for _, id := range je.participants { + refid = id + vre.controllers[id].Stop() + } + + dbClient := vre.dbClientFactory() + if err := dbClient.Connect(); err != nil { + log.Errorf("transitionJournal: unable to connect to the database: %v", err) + return + } + defer dbClient.Close() + + if err := dbClient.Begin(); err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + + params, err := readRow(dbClient, refid) + if err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + var newids []int + for _, sgtid := range je.journal.ShardGtids { + bls := vre.controllers[refid].source + bls.Keyspace, bls.Shard = sgtid.Keyspace, sgtid.Shard + query := fmt.Sprintf("insert into _vt.vreplication "+ + "(workflow, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state, db_name) "+ + "values (%v, %v, %v, %v, %v, %v, %v, 0, '%v', %v)", + encodeString(params["workflow"]), encodeString(bls.String()), encodeString(sgtid.Gtid), params["max_tps"], params["max_replication_lag"], encodeString(params["tablet_types"]), time.Now().Unix(), binlogplayer.BlpRunning, encodeString(vre.dbName)) + qr, err := vre.executeFetchMaybeCreateTable(dbClient, query, 1) + if err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + log.Infof("Created stream: %v for %v", qr.InsertID, sgtid) + newids = append(newids, int(qr.InsertID)) + } + for _, id := range je.participants { + _, err := vre.executeFetchMaybeCreateTable(dbClient, binlogplayer.DeleteVReplication(uint32(id)), 1) + if err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + log.Infof("Deleted stream: %v", id) + } + if err := dbClient.Commit(); err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + + for _, id := range je.participants { + delete(vre.controllers, id) + } + + for _, id := range newids { + params, err := readRow(dbClient, id) + if err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil, vre) + if err != nil { + log.Errorf("transitionJournal: %v", err) + return + } + vre.controllers[id] = ct + } + log.Infof("Completed transition for journal:workload %v", key) +} + // WaitForPos waits for the replication to reach the specified position. func (vre *Engine) WaitForPos(ctx context.Context, id int, pos string) error { start := time.Now() diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 36b3e55c693..24853187a62 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -19,6 +19,7 @@ package vreplication import ( "flag" "fmt" + "io" "os" "reflect" "regexp" @@ -82,7 +83,7 @@ func TestMain(m *testing.M) { // engines cannot be initialized in testenv because it introduces // circular dependencies. streamerEngine = vstreamer.NewEngine(env.SrvTopo, env.SchemaEngine) - streamerEngine.InitDBConfig(env.Dbcfgs) + streamerEngine.InitDBConfig(env.Dbcfgs.DbaWithDB()) streamerEngine.Open(env.KeyspaceName, env.Cells[0]) defer streamerEngine.Close() @@ -96,6 +97,8 @@ func TestMain(m *testing.M) { return 1 } + InitVStreamerClient(env.Dbcfgs) + playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, vrepldb) if err := playerEngine.Open(context.Background()); err != nil { fmt.Fprintf(os.Stderr, "%v", err) @@ -122,6 +125,22 @@ func resetBinlogClient() { globalFBC = &fakeBinlogClient{} } +func masterPosition(t *testing.T) string { + t.Helper() + pos, err := env.Mysqld.MasterPosition() + if err != nil { + t.Fatal(err) + } + return mysql.EncodePosition(pos) +} + +func execStatements(t *testing.T, queries []string) { + t.Helper() + if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + t.Error(err) + } +} + //-------------------------------------- // Topos and tablets @@ -145,6 +164,26 @@ func addTablet(id int) *topodatapb.Tablet { return tablet } +func addOtherTablet(id int, keyspace, shard string) *topodatapb.Tablet { + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: env.Cells[0], + Uid: uint32(id), + }, + Keyspace: keyspace, + Shard: shard, + KeyRange: &topodatapb.KeyRange{}, + Type: topodatapb.TabletType_REPLICA, + PortMap: map[string]int32{ + "test": int32(id), + }, + } + if err := env.TopoServ.CreateTablet(context.Background(), tablet); err != nil { + panic(err) + } + return tablet +} + func deleteTablet(tablet *topodatapb.Tablet) { env.TopoServ.DeleteTablet(context.Background(), tablet.Alias) // This is not automatically removed from shard replication, which results in log spam. @@ -174,6 +213,10 @@ func (ftc *fakeTabletConn) StreamHealth(ctx context.Context, callback func(*quer // VStream directly calls into the pre-initialized engine. func (ftc *fakeTabletConn) VStream(ctx context.Context, target *querypb.Target, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + if target.Keyspace != "vttest" { + <-ctx.Done() + return io.EOF + } return streamerEngine.Stream(ctx, startPos, filter, send) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/journal_test.go b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go new file mode 100644 index 00000000000..32402c1fdb4 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go @@ -0,0 +1,336 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "testing" + + "golang.org/x/net/context" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" +) + +func TestJournalOneToOne(t *testing.T) { + defer deleteTablet(addTablet(100)) + defer deleteTablet(addOtherTablet(101, "other_keyspace", "0")) + + execStatements(t, []string{ + "create table t(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t", + fmt.Sprintf("drop table %s.t", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, firstID := startVReplication(t, bls, "") + + journal := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_SHARDS, + Participants: []*binlogdatapb.KeyspaceShard{{ + Keyspace: "vttest", + Shard: "0", + }}, + ShardGtids: []*binlogdatapb.ShardGtid{{ + Keyspace: "other_keyspace", + Shard: "0", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10", + }}, + } + query := fmt.Sprintf("insert into _vt.resharding_journal(id, db_name, val) values (1, 'vttest', %v)", encodeString(journal.String())) + execStatements(t, []string{createReshardingJournalTable, query}) + defer execStatements(t, []string{"delete from _vt.resharding_journal"}) + + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + "begin", + `/insert into _vt.vreplication.*workflow, source, pos.*values.*'test', 'keyspace:\\"other_keyspace\\" shard:\\"0\\.*'MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10'`, + fmt.Sprintf("delete from _vt.vreplication where id=%d", firstID), + "commit", + "/update _vt.vreplication set state='Running', message='' where id.*", + }) + + // Delete all vreplication streams. There should be only one, but we don't know its id. + if _, err := playerEngine.Exec("delete from _vt.vreplication"); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) +} + +func TestJournalOneToMany(t *testing.T) { + defer deleteTablet(addTablet(100)) + defer deleteTablet(addOtherTablet(101, "other_keyspace", "-80")) + defer deleteTablet(addOtherTablet(102, "other_keyspace", "80-")) + + execStatements(t, []string{ + "create table t(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t", + fmt.Sprintf("drop table %s.t", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, firstID := startVReplication(t, bls, "") + + journal := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_SHARDS, + Participants: []*binlogdatapb.KeyspaceShard{{ + Keyspace: "vttest", + Shard: "0", + }}, + ShardGtids: []*binlogdatapb.ShardGtid{{ + Keyspace: "other_keyspace", + Shard: "-80", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-5", + }, { + Keyspace: "other_keyspace", + Shard: "80-", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:5-10", + }}, + } + query := fmt.Sprintf("insert into _vt.resharding_journal(id, db_name, val) values (1, 'vttest', %v)", encodeString(journal.String())) + execStatements(t, []string{createReshardingJournalTable, query}) + defer execStatements(t, []string{"delete from _vt.resharding_journal"}) + + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + "begin", + `/insert into _vt.vreplication.*workflow, source, pos.*values.*'test', 'keyspace:\\"other_keyspace\\" shard:\\"-80\\.*'MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-5'`, + `/insert into _vt.vreplication.*workflow, source, pos.*values.*'test', 'keyspace:\\"other_keyspace\\" shard:\\"80-\\.*'MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:5-10'`, + fmt.Sprintf("delete from _vt.vreplication where id=%d", firstID), + "commit", + "/update _vt.vreplication set state='Running', message='' where id.*", + "/update _vt.vreplication set state='Running', message='' where id.*", + }) + + // Delete all vreplication streams. There should be only one, but we don't know its id. + if _, err := playerEngine.Exec("delete from _vt.vreplication"); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) +} + +func TestJournalTablePresent(t *testing.T) { + defer deleteTablet(addTablet(100)) + defer deleteTablet(addOtherTablet(101, "other_keyspace", "0")) + + execStatements(t, []string{ + "create table t(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t", + fmt.Sprintf("drop table %s.t", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + _, firstID := startVReplication(t, bls, "") + + journal := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_TABLES, + Participants: []*binlogdatapb.KeyspaceShard{{ + Keyspace: "vttest", + Shard: "0", + }}, + Tables: []string{"t"}, + ShardGtids: []*binlogdatapb.ShardGtid{{ + Keyspace: "other_keyspace", + Shard: "0", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10", + }}, + } + query := fmt.Sprintf("insert into _vt.resharding_journal(id, db_name, val) values (1, 'vttest', %v)", encodeString(journal.String())) + execStatements(t, []string{createReshardingJournalTable, query}) + defer execStatements(t, []string{"delete from _vt.resharding_journal"}) + + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + "begin", + `/insert into _vt.vreplication.*workflow, source, pos.*values.*'test', 'keyspace:\\"other_keyspace\\" shard:\\"0\\.*'MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10'`, + fmt.Sprintf("delete from _vt.vreplication where id=%d", firstID), + "commit", + "/update _vt.vreplication set state='Running', message='' where id.*", + }) + + // Delete all vreplication streams. There should be only one, but we don't know its id. + if _, err := playerEngine.Exec("delete from _vt.vreplication"); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) +} + +func TestJournalTableNotPresent(t *testing.T) { + defer deleteTablet(addTablet(100)) + defer deleteTablet(addOtherTablet(101, "other_keyspace", "0")) + + execStatements(t, []string{ + "create table t(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t", + fmt.Sprintf("drop table %s.t", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, _ = startVReplication(t, bls, "") + + journal := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_TABLES, + Participants: []*binlogdatapb.KeyspaceShard{{ + Keyspace: "vttest", + Shard: "0", + }}, + Tables: []string{"t1"}, + ShardGtids: []*binlogdatapb.ShardGtid{{ + Keyspace: "other_keyspace", + Shard: "0", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10", + }}, + } + query := fmt.Sprintf("insert into _vt.resharding_journal(id, db_name, val) values (1, 'vttest', %v)", encodeString(journal.String())) + execStatements(t, []string{createReshardingJournalTable, query}) + defer execStatements(t, []string{"delete from _vt.resharding_journal"}) + + // Wait for a heartbeat based update to confirm that the existing vreplication was not transitioned. + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + }) + + // Delete all vreplication streams. There should be only one, but we don't know its id. + if _, err := playerEngine.Exec("delete from _vt.vreplication"); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) +} + +func TestJournalTableMixed(t *testing.T) { + defer deleteTablet(addTablet(100)) + defer deleteTablet(addOtherTablet(101, "other_keyspace", "0")) + + execStatements(t, []string{ + "create table t(id int, val varbinary(128), primary key(id))", + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t(id int, val varbinary(128), primary key(id))", vrepldb), + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t", + "drop table t1", + fmt.Sprintf("drop table %s.t", vrepldb), + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t", + }, { + Match: "t1", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + _, _ = startVReplication(t, bls, "") + + journal := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_TABLES, + Participants: []*binlogdatapb.KeyspaceShard{{ + Keyspace: "vttest", + Shard: "0", + }}, + Tables: []string{"t"}, + ShardGtids: []*binlogdatapb.ShardGtid{{ + Keyspace: "other_keyspace", + Shard: "0", + Gtid: "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10", + }}, + } + query := fmt.Sprintf("insert into _vt.resharding_journal(id, db_name, val) values (1, 'vttest', %v)", encodeString(journal.String())) + execStatements(t, []string{createReshardingJournalTable, query}) + defer execStatements(t, []string{"delete from _vt.resharding_journal"}) + + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + "/update _vt.vreplication set state='Stopped', message='unable to handle journal event: tables were partially matched' where id", + }) + + // Delete all vreplication streams. There should be only one, but we don't know its id. + if _, err := playerEngine.Exec("delete from _vt.vreplication"); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 215f769155b..488f325ef0f 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -461,6 +461,56 @@ func TestBuildPlayerPlan(t *testing.T) { }, }, }, + }, { + // keyspace_id + input: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2, keyspace_id() ksid from t1", + }}, + }, + plan: &TestReplicatorPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2, keyspace_id() from t1", + }}, + }, + TargetTables: []string{"t1"}, + TablePlans: map[string]*TestTablePlan{ + "t1": { + TargetName: "t1", + SendRule: "t1", + PKReferences: []string{"c1"}, + InsertFront: "insert into t1(c1,c2,ksid)", + InsertValues: "(:a_c1,:a_c2,:a_keyspace_id)", + Insert: "insert into t1(c1,c2,ksid) values (:a_c1,:a_c2,:a_keyspace_id)", + Update: "update t1 set c2=:a_c2, ksid=:a_keyspace_id where c1=:b_c1", + Delete: "delete from t1 where c1=:b_c1", + }, + }, + }, + planpk: &TestReplicatorPlan{ + VStreamFilter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select c1, c2, keyspace_id(), pk1, pk2 from t1", + }}, + }, + TargetTables: []string{"t1"}, + TablePlans: map[string]*TestTablePlan{ + "t1": { + TargetName: "t1", + SendRule: "t1", + PKReferences: []string{"c1", "pk1", "pk2"}, + InsertFront: "insert into t1(c1,c2,ksid)", + InsertValues: "(:a_c1,:a_c2,:a_keyspace_id)", + Insert: "insert into t1(c1,c2,ksid) select :a_c1, :a_c2, :a_keyspace_id from dual where (:a_pk1,:a_pk2) <= (1,'aaa')", + Update: "update t1 set c2=:a_c2, ksid=:a_keyspace_id where c1=:b_c1 and (:b_pk1,:b_pk2) <= (1,'aaa')", + Delete: "delete from t1 where c1=:b_c1 and (:b_pk1,:b_pk2) <= (1,'aaa')", + }, + }, + }, }, { // syntax error input: &binlogdatapb.Filter{ diff --git a/go/vt/vttablet/tabletmanager/vreplication/stats.go b/go/vt/vttablet/tabletmanager/vreplication/stats.go index e6a2039bb2f..be66fae9997 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/stats.go +++ b/go/vt/vttablet/tabletmanager/vreplication/stats.go @@ -73,6 +73,35 @@ func (st *vrStats) register() { } return result }) + + stats.NewCounterFunc( + "VReplicationTotalSecondsBehindMaster", + "vreplication seconds behind master aggregated across all streams", + func() int64 { + st.mu.Lock() + defer st.mu.Unlock() + result := int64(0) + for _, ct := range st.controllers { + result += ct.blpStats.SecondsBehindMaster.Get() + } + return result + }) + + stats.NewRateFunc( + "VReplicationQPS", + "vreplication operations per second aggregated across all streams", + func() map[string][]float64 { + st.mu.Lock() + defer st.mu.Unlock() + result := make(map[string][]float64) + for _, ct := range st.controllers { + for k, v := range ct.blpStats.Rates.Get() { + result[k] = v + } + } + return result + }) + stats.Publish("VReplicationSource", stats.StringMapFunc(func() map[string]string { st.mu.Lock() defer st.mu.Unlock() @@ -193,5 +222,71 @@ var vreplicationTemplate = ` {{range $key, $values := .Rates}}{{$key}}: {{range $values}}{{.}} {{end}}
{{end}} {{range $index, $value := .Messages}}{{$value}}
{{end}} {{end}} +
QPS All Streams
+ + + {{else}}VReplication is closed.{{end}} ` diff --git a/go/vt/vttablet/tabletmanager/vreplication/stats_test.go b/go/vt/vttablet/tabletmanager/vreplication/stats_test.go index b46e1c4c4d5..8dc9e599c2e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/stats_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/stats_test.go @@ -19,6 +19,7 @@ package vreplication import ( "bytes" "html/template" + "strings" "testing" "time" @@ -111,7 +112,7 @@ func TestStatusHtml(t *testing.T) { tpl := template.Must(template.New("test").Parse(vreplicationTemplate)) buf := bytes.NewBuffer(nil) tpl.Execute(buf, testStats.status()) - if buf.String() != wantOut { + if strings.Contains(buf.String(), wantOut) { t.Errorf("output: %v, want %v", buf, wantOut) } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index eed3576ff80..95104a0a944 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -49,7 +49,9 @@ type colExpr struct { // operation==opCount: nothing is set. // operation==opSum: for 'sum(a)', expr is set to 'a'. operation operation - expr sqlparser.Expr + // expr stores the expected field name from vstreamer and dictates + // the generated bindvar names, like a_col or b_col. + expr sqlparser.Expr // references contains all the column names referenced in the expression. references map[string]bool @@ -332,6 +334,14 @@ func (tpb *tablePlanBuilder) analyzeExpr(selExpr sqlparser.SelectExpr) (*colExpr tpb.addCol(innerCol.Name) cexpr.references[innerCol.Name.Lowered()] = true return cexpr, nil + case "keyspace_id": + if len(expr.Exprs) != 0 { + return nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + tpb.sendSelect.SelectExprs = append(tpb.sendSelect.SelectExprs, &sqlparser.AliasedExpr{Expr: aliased.Expr}) + // The vstreamer responds with "keyspace_id" as the field name for this request. + cexpr.expr = &sqlparser.ColName{Name: sqlparser.NewColIdent("keyspace_id")} + return cexpr, nil } } err := sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 37152f006e8..b663efe6e03 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -30,10 +30,8 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vttablet/tabletconn" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" @@ -176,21 +174,15 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma return fmt.Errorf("plan not found for table: %s, current plans are: %#v", tableName, plan.TargetTables) } - vsClient, err := tabletconn.GetDialer()(vc.vr.sourceTablet, grpcclient.FailFast(false)) + err = vc.vr.sourceVStreamer.Open(ctx) if err != nil { - return fmt.Errorf("error dialing tablet: %v", err) + return fmt.Errorf("error opening vsclient: %v", err) } - defer vsClient.Close(ctx) + defer vc.vr.sourceVStreamer.Close(ctx) ctx, cancel := context.WithTimeout(ctx, copyTimeout) defer cancel() - target := &querypb.Target{ - Keyspace: vc.vr.sourceTablet.Keyspace, - Shard: vc.vr.sourceTablet.Shard, - TabletType: vc.vr.sourceTablet.Type, - } - var lastpkpb *querypb.QueryResult if lastpkqr := copyState[tableName]; lastpkqr != nil { lastpkpb = sqltypes.ResultToProto3(lastpkqr) @@ -198,7 +190,7 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma var pkfields []*querypb.Field var updateCopyState *sqlparser.ParsedQuery - err = vsClient.VStreamRows(ctx, target, initialPlan.SendRule.Filter, lastpkpb, func(rows *binlogdatapb.VStreamRowsResponse) error { + err = vc.vr.sourceVStreamer.VStreamRows(ctx, initialPlan.SendRule.Filter, lastpkpb, func(rows *binlogdatapb.VStreamRowsResponse) error { select { case <-ctx.Done(): return io.EOF diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 3a4bfd1fff4..511bf716dde 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -28,12 +28,9 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/vttablet/tabletconn" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - querypb "vitess.io/vitess/go/vt/proto/query" ) type vplayer struct { @@ -56,6 +53,8 @@ type vplayer struct { lastTimestampNs int64 // timeOffsetNs keeps track of the clock difference with respect to source tablet. timeOffsetNs int64 + // canAcceptStmtEvents set to true if the current player can accept events in statement mode. Only true for filters that are match all. + canAcceptStmtEvents bool } func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { @@ -91,6 +90,15 @@ func (vp *vplayer) play(ctx context.Context) error { } vp.replicatorPlan = plan + // We can't run in statement mode if there are filters defined. + vp.canAcceptStmtEvents = true + for _, rule := range vp.vr.source.Filter.Rules { + if rule.Filter != "" || rule.Match != "/.*" { + vp.canAcceptStmtEvents = false + break + } + } + if err := vp.fetchAndApply(ctx); err != nil { msg := err.Error() vp.vr.stats.History.Add(&binlogplayer.StatsHistoryRecord{ @@ -105,28 +113,23 @@ func (vp *vplayer) play(ctx context.Context) error { return nil } -func (vp *vplayer) fetchAndApply(ctx context.Context) error { - log.Infof("Starting VReplication player id: %v, startPos: %v, stop: %v, source: %v, filter: %v", vp.vr.id, vp.startPos, vp.stopPos, vp.vr.sourceTablet, vp.vr.source) +func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { + log.Infof("Starting VReplication player id: %v, startPos: %v, stop: %v, filter: %v", vp.vr.id, vp.startPos, vp.stopPos, vp.vr.source) - vsClient, err := tabletconn.GetDialer()(vp.vr.sourceTablet, grpcclient.FailFast(false)) + err = vp.vr.sourceVStreamer.Open(ctx) if err != nil { - return fmt.Errorf("error dialing tablet: %v", err) + return fmt.Errorf("error creating vstreamer client: %v", err) } - defer vsClient.Close(ctx) + defer vp.vr.sourceVStreamer.Close(ctx) + ctx, cancel := context.WithCancel(ctx) defer cancel() relay := newRelayLog(ctx, relayLogMaxItems, relayLogMaxSize) - target := &querypb.Target{ - Keyspace: vp.vr.sourceTablet.Keyspace, - Shard: vp.vr.sourceTablet.Shard, - TabletType: vp.vr.sourceTablet.Type, - } - log.Infof("Sending vstream command: %v", vp.replicatorPlan.VStreamFilter) streamErr := make(chan error, 1) go func() { - streamErr <- vsClient.VStream(ctx, target, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { + streamErr <- vp.vr.sourceVStreamer.VStream(ctx, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { return relay.Send(events) }) }() @@ -143,8 +146,10 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) error { cancel() <-streamErr }() + // If the apply thread ends with io.EOF, it means either the Engine - // is shutting down and canceled the context, or stop position was reached. + // is shutting down and canceled the context, or stop position was reached, + // or a journal event was encountered. // If so, we return nil which will cause the controller to not retry. if err == io.EOF { return nil @@ -171,6 +176,14 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) error { } } +func (vp *vplayer) applyStmtEvent(ctx context.Context, event *binlogdatapb.VEvent) error { + if vp.canAcceptStmtEvents { + _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml) + return err + } + return fmt.Errorf("filter rules are not supported for SBR replication: %v", vp.vr.source.Filter.GetRules()) +} + func (vp *vplayer) applyRowEvent(ctx context.Context, rowEvent *binlogdatapb.RowEvent) error { tplan := vp.tablePlans[rowEvent.TableName] if tplan == nil { @@ -196,7 +209,7 @@ func (vp *vplayer) updatePos(ts int64) (posReached bool, err error) { vp.unsavedEvent = nil vp.timeLastSaved = time.Now() vp.vr.stats.SetLastPosition(vp.pos) - posReached = !vp.stopPos.IsZero() && vp.pos.Equal(vp.stopPos) + posReached = !vp.stopPos.IsZero() && vp.pos.AtLeast(vp.stopPos) if posReached { if vp.saveStop { if err := vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stopped at position %v", vp.stopPos)); err != nil { @@ -256,7 +269,7 @@ func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { mustSave := false switch event.Type { case binlogdatapb.VEventType_COMMIT: - if vp.pos.Equal(vp.stopPos) { + if !vp.stopPos.IsZero() && vp.pos.AtLeast(vp.stopPos) { mustSave = true break } @@ -303,15 +316,6 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m if vp.stopPos.IsZero() { return nil } - if !vp.pos.Equal(vp.stopPos) && vp.pos.AtLeast(vp.stopPos) { - // Code is unreachable, but bad data can cause this to happen. - if vp.saveStop { - if err := vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("next event position %v exceeds stop pos %v, exiting without applying", vp.pos, vp.stopPos)); err != nil { - return err - } - } - return io.EOF - } case binlogdatapb.VEventType_BEGIN: // No-op: begin is called as needed. case binlogdatapb.VEventType_COMMIT: @@ -345,13 +349,32 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return err } vp.tablePlans[event.FieldEvent.TableName] = tplan + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE, binlogdatapb.VEventType_REPLACE: + // This is a player using stament based replication + if err := vp.vr.dbClient.Begin(); err != nil { + return err + } + + if err := vp.applyStmtEvent(ctx, event); err != nil { + return err + } case binlogdatapb.VEventType_ROW: + // This player is configured for row based replication if err := vp.vr.dbClient.Begin(); err != nil { return err } if err := vp.applyRowEvent(ctx, event.RowEvent); err != nil { return err } + case binlogdatapb.VEventType_OTHER: + // Just update the position. + posReached, err := vp.updatePos(event.Timestamp) + if err != nil { + return err + } + if posReached { + return io.EOF + } case binlogdatapb.VEventType_DDL: if vp.vr.dbClient.InTransaction { return fmt.Errorf("unexpected state: DDL encountered in the middle of a transaction: %v", event.Ddl) @@ -403,6 +426,47 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return io.EOF } } + case binlogdatapb.VEventType_JOURNAL: + // Ensure that we don't have a partial set of table matches in the journal. + switch event.Journal.MigrationType { + case binlogdatapb.MigrationType_SHARDS: + // All tables of the source were migrated. So, no validation needed. + case binlogdatapb.MigrationType_TABLES: + // Validate that all or none of the tables are in the journal. + jtables := make(map[string]bool) + for _, table := range event.Journal.Tables { + jtables[table] = true + } + found := false + notFound := false + for tableName := range vp.replicatorPlan.TablePlans { + if _, ok := jtables[tableName]; ok { + found = true + } else { + notFound = true + } + } + switch { + case found && notFound: + // Some were found and some were not found. We can't handle this. + if err := vp.vr.setState(binlogplayer.BlpStopped, "unable to handle journal event: tables were partially matched"); err != nil { + return err + } + return io.EOF + case notFound: + // None were found. Ignore journal. + return nil + } + // All were found. We must register journal. + } + + if err := vp.vr.vre.registerJournal(event.Journal, int(vp.vr.id)); err != nil { + if err := vp.vr.setState(binlogplayer.BlpStopped, err.Error()); err != nil { + return err + } + return io.EOF + } + return io.EOF case binlogdatapb.VEventType_HEARTBEAT: // No-op: heartbeat timings are calculated in outer loop. } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index 10f66487e05..d84bfb3fdc0 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -26,13 +26,99 @@ import ( "golang.org/x/net/context" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) +func TestPlayerStatementModeWithFilter(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table src1", + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + input := []string{ + "set @@session.binlog_format='STATEMENT'", + "insert into src1 values(1, 'aaa')", + "set @@session.binlog_format='ROW'", + } + + // It does not work when filter is enabled + output := []string{ + "begin", + "/update _vt.vreplication set message='filter rules are not supported for SBR", + } + + execStatements(t, input) + expectDBClientQueries(t, output) +} + +func TestPlayerStatementMode(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.src1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + Filter: "", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + input := []string{ + "set @@session.binlog_format='STATEMENT'", + "insert into src1 values(1, 'aaa')", + "set @@session.binlog_format='ROW'", + } + + output := []string{ + "begin", + "insert into src1 values(1, 'aaa')", + "/update _vt.vreplication set pos=", + "commit", + } + + execStatements(t, input) + expectDBClientQueries(t, output) +} + func TestPlayerFilters(t *testing.T) { defer deleteTablet(addTablet(100)) @@ -80,7 +166,13 @@ func TestPlayerFilters(t *testing.T) { Match: "/nopk", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -314,7 +406,15 @@ func TestPlayerKeywordNames(t *testing.T) { Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -447,6 +547,88 @@ func TestPlayerKeywordNames(t *testing.T) { } } } + +var shardedVSchema = `{ + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "src1": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + } + } +}` + +func TestPlayerKeyspaceID(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.dst1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + if err := env.SetVSchema(shardedVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select id, keyspace_id() as val from src1", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + testcases := []struct { + input string + output []string + table string + data [][]string + }{{ + // insert with insertNormal + input: "insert into src1 values(1, 'aaa')", + output: []string{ + "begin", + "insert into dst1(id,val) values (1,'\x16k@\xb4J\xbaK\xd6')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "\x16k@\xb4J\xbaK\xd6"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + expectDBClientQueries(t, tcases.output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + func TestUnicode(t *testing.T) { defer deleteTablet(addTablet(100)) @@ -466,7 +648,13 @@ func TestUnicode(t *testing.T) { Filter: "select * from src1", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -533,7 +721,13 @@ func TestPlayerUpdates(t *testing.T) { Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -642,7 +836,13 @@ func TestPlayerRowMove(t *testing.T) { Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -717,7 +917,13 @@ func TestPlayerTypes(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { input string @@ -808,8 +1014,13 @@ func TestPlayerDDL(t *testing.T) { Match: "/.*", }}, } - - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") // Issue a dummy change to ensure vreplication is initialized. Otherwise there // is a race between the DDLs and the schema loader of vstreamer. // Root cause seems to be with MySQL where t1 shows up in information_schema before @@ -829,8 +1040,13 @@ func TestPlayerDDL(t *testing.T) { "/update _vt.vreplication set pos=", }) cancel() - - cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_STOP, + } + cancel, id := startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val varchar(128)"}) pos1 := masterPosition(t) execStatements(t, []string{"alter table t1 drop column val"}) @@ -857,13 +1073,20 @@ func TestPlayerDDL(t *testing.T) { "commit", }) cancel() - + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC, + } execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + cancel, _ = startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) expectDBClientQueries(t, []string{ "alter table t1 add column val1 varchar(128)", "/update _vt.vreplication set pos=", + // The apply of the DDL on target generates an "other" event. + "/update _vt.vreplication set pos=", }) execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) expectDBClientQueries(t, []string{ @@ -878,12 +1101,20 @@ func TestPlayerDDL(t *testing.T) { fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), }) + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC_IGNORE, + } execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") + cancel, _ = startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) expectDBClientQueries(t, []string{ "alter table t1 add column val1 varchar(128)", "/update _vt.vreplication set pos=", + // The apply of the DDL on target generates an "other" event. + "/update _vt.vreplication set pos=", }) execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) expectDBClientQueries(t, []string{ @@ -1012,7 +1243,13 @@ func TestPlayerIdleUpdate(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1058,7 +1295,13 @@ func TestPlayerSplitTransaction(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1096,7 +1339,13 @@ func TestPlayerLockErrors(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1170,7 +1419,13 @@ func TestPlayerCancelOnLock(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1242,7 +1497,13 @@ func TestPlayerBatching(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1304,6 +1565,9 @@ func TestPlayerBatching(t *testing.T) { "/update _vt.vreplication set pos=", "alter table t1 drop column val2", "/update _vt.vreplication set pos=", + // The apply of the DDLs on target generates two "other" event. + "/update _vt.vreplication set pos=", + "/update _vt.vreplication set pos=", }) } @@ -1339,7 +1603,13 @@ func TestPlayerRelayLogMaxSize(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1428,7 +1698,13 @@ func TestRestartOnVStreamEnd(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1479,7 +1755,14 @@ func TestTimestamp(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") @@ -1504,22 +1787,9 @@ func TestTimestamp(t *testing.T) { expectData(t, "t1", [][]string{{"1", want, want}}) } -func execStatements(t *testing.T, queries []string) { - t.Helper() - if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { - t.Error(err) - } -} - -func startVReplication(t *testing.T, filter *binlogdatapb.Filter, onddl binlogdatapb.OnDDLAction, pos string) (cancelFunc func(), id int) { +func startVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) (cancelFunc func(), id int) { t.Helper() - bls := &binlogdatapb.BinlogSource{ - Keyspace: env.KeyspaceName, - Shard: env.ShardName, - Filter: filter, - OnDdl: onddl, - } if pos == "" { pos = masterPosition(t) } @@ -1545,12 +1815,3 @@ func startVReplication(t *testing.T, filter *binlogdatapb.Filter, onddl binlogda }) }, int(qr.InsertID) } - -func masterPosition(t *testing.T) string { - t.Helper() - pos, err := env.Mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } - return mysql.EncodePosition(pos) -} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index d067191c279..a736920c741 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -30,7 +30,6 @@ import ( "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) var ( @@ -45,29 +44,35 @@ var ( replicaLagTolerance = 10 * time.Second ) +// vreplicator provides the core logic to start vreplication streams type vreplicator struct { - id uint32 - source *binlogdatapb.BinlogSource - sourceTablet *topodatapb.Tablet - stats *binlogplayer.Stats - dbClient *vdbClient + vre *Engine + id uint32 + dbClient *vdbClient + // source + source *binlogdatapb.BinlogSource + sourceVStreamer VStreamerClient + + stats *binlogplayer.Stats // mysqld is used to fetch the local schema. - mysqld mysqlctl.MysqlDaemon - + mysqld mysqlctl.MysqlDaemon tableKeys map[string][]string } -func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceTablet *topodatapb.Tablet, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vreplicator { +// newVReplicator creates a new vreplicator +func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon, vre *Engine) *vreplicator { return &vreplicator{ - id: id, - source: source, - sourceTablet: sourceTablet, - stats: stats, - dbClient: newVDBClient(dbClient, stats), - mysqld: mysqld, + vre: vre, + id: id, + source: source, + sourceVStreamer: sourceVStreamer, + stats: stats, + dbClient: newVDBClient(dbClient, stats), + mysqld: mysqld, } } +// Replicate starts a vreplication stream. func (vr *vreplicator) Replicate(ctx context.Context) error { tableKeys, err := vr.buildTableKeys() if err != nil { @@ -78,12 +83,13 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { for { settings, numTablesToCopy, err := vr.readSettings(ctx) if err != nil { - return fmt.Errorf("error reading VReplication settings: %v", err) + return err } // If any of the operations below changed state to Stopped, we should return. if settings.State == binlogplayer.BlpStopped { return nil } + switch { case numTablesToCopy != 0: if err := newVCopier(vr).copyNext(ctx, settings); err != nil { @@ -158,7 +164,7 @@ func (vr *vreplicator) setMessage(message string) error { Time: time.Now(), Message: message, }) - query := fmt.Sprintf("update _vt.vreplication set message=%v where id=%v", encodeString(message), vr.id) + query := fmt.Sprintf("update _vt.vreplication set message=%v where id=%v", encodeString(binlogplayer.MessageTruncate(message)), vr.id) if _, err := vr.dbClient.Execute(query); err != nil { return fmt.Errorf("could not set message: %v: %v", query, err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go new file mode 100644 index 00000000000..3f303238208 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -0,0 +1,220 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "errors" + "fmt" + "sync" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/vtgate/vindexes" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +var ( + _ VStreamerClient = (*TabletVStreamerClient)(nil) + _ VStreamerClient = (*MySQLVStreamerClient)(nil) + dbcfgs *dbconfigs.DBConfigs +) + +// VStreamerClient exposes the core interface of a vstreamer +type VStreamerClient interface { + // Open sets up all the environment for a vstream + Open(ctx context.Context) error + // Close closes a vstream + Close(ctx context.Context) error + + // VStream streams VReplication events based on the specified filter. + VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error + + // VStreamRows streams rows of a table from the specified starting point. + VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error +} + +// TabletVStreamerClient a vstream client backed by vttablet +type TabletVStreamerClient struct { + // mu protects isOpen, streamers, streamIdx and kschema. + mu sync.Mutex + + isOpen bool + + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService +} + +// MySQLVStreamerClient a vstream client backed by MySQL +type MySQLVStreamerClient struct { + // mu protects isOpen, streamers, streamIdx and kschema. + mu sync.Mutex + + isOpen bool + + sourceConnParams *mysql.ConnParams + sourceSe *schema.Engine +} + +// NewTabletVStreamerClient creates a new TabletVStreamerClient +func NewTabletVStreamerClient(tablet *topodatapb.Tablet) *TabletVStreamerClient { + return &TabletVStreamerClient{ + tablet: tablet, + target: &querypb.Target{ + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + TabletType: tablet.Type, + }, + } +} + +// Open part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) Open(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if vsClient.isOpen { + return nil + } + vsClient.isOpen = true + + vsClient.tsQueryService, err = tabletconn.GetDialer()(vsClient.tablet, grpcclient.FailFast(false)) + return err +} + +// Close part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) Close(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if !vsClient.isOpen { + return nil + } + vsClient.isOpen = false + return vsClient.tsQueryService.Close(ctx) +} + +// VStream part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + if !vsClient.isOpen { + return errors.New("can't VStream without opening client") + } + return vsClient.tsQueryService.VStream(ctx, vsClient.target, startPos, filter, send) +} + +// VStreamRows part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error { + if !vsClient.isOpen { + return errors.New("can't VStreamRows without opening client") + } + return vsClient.tsQueryService.VStreamRows(ctx, vsClient.target, query, lastpk, send) +} + +// NewMySQLVStreamerClient is a vstream client that allows you to stream directly from MySQL. +// In order to achieve this, the following creates a vstreamer Engine with a dummy in memorytopo. +func NewMySQLVStreamerClient() *MySQLVStreamerClient { + if dbcfgs == nil { + panic("can't use MySQLVStreamerClient without calling InitVStreamerClient() ") + } + // TODO: For now external mysql streams can only be used with ExternalReplWithDB creds. + // In the future we will support multiple users. + vsClient := &MySQLVStreamerClient{ + sourceConnParams: dbcfgs.ExternalReplWithDB(), + } + return vsClient +} + +// Open part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if vsClient.isOpen { + return nil + } + vsClient.isOpen = true + + // Let's create all the required components by vstreamer + + vsClient.sourceSe = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) + vsClient.sourceSe.InitDBConfig(vsClient.sourceConnParams) + err = vsClient.sourceSe.Open() + if err != nil { + return err + } + return nil +} + +// Close part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) Close(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if !vsClient.isOpen { + return nil + } + + vsClient.isOpen = false + vsClient.sourceSe.Close() + return nil +} + +// VStream part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + if !vsClient.isOpen { + return errors.New("can't VStream without opening client") + } + streamer := vstreamer.NewVStreamer(ctx, vsClient.sourceConnParams, vsClient.sourceSe, startPos, filter, &vindexes.KeyspaceSchema{}, send) + return streamer.Stream() +} + +// VStreamRows part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error { + if !vsClient.isOpen { + return errors.New("can't VStreamRows without opening client") + } + var row []sqltypes.Value + if lastpk != nil { + r := sqltypes.Proto3ToResult(lastpk) + if len(r.Rows) != 1 { + return fmt.Errorf("unexpected lastpk input: %v", lastpk) + } + row = r.Rows[0] + } + streamer := vstreamer.NewRowStreamer(ctx, vsClient.sourceConnParams, vsClient.sourceSe, query, row, &vindexes.KeyspaceSchema{}, send) + return streamer.Stream() +} + +// InitVStreamerClient initializes config for vstreamer client +func InitVStreamerClient(cfg *dbconfigs.DBConfigs) { + dbcfgs = cfg +} + +type checker struct{} + +var _ = connpool.MySQLChecker(checker{}) + +func (checker) CheckMySQL() {} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go new file mode 100644 index 00000000000..e63bad6e668 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -0,0 +1,529 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "reflect" + "strings" + "testing" + "time" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +func TestTabletVStreamerClientOpen(t *testing.T) { + tablet := addTablet(100) + defer deleteTablet(tablet) + + type fields struct { + isOpen bool + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "initializes streamer client", + fields: fields{ + tablet: tablet, + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &TabletVStreamerClient{ + tablet: tcase.fields.tablet, + } + + err := vsClient.Open(tcase.args.ctx) + + if err != nil { + if !strings.Contains(err.Error(), tcase.err) { + t.Errorf("TabletVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + return + } + + if tcase.err != "" { + t.Errorf("TabletVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + + if !vsClient.isOpen { + t.Errorf("TabletVStreamerClient.Open() isOpen set to false, expected true") + } + + if vsClient.tablet == nil { + t.Errorf("TabletVStreamerClient.Open() expected sourceSe to be set") + } + }) + } +} + +func TestTabletVStreamerClientClose(t *testing.T) { + tablet := addTablet(100) + defer deleteTablet(tablet) + + type fields struct { + isOpen bool + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "closes engine correctly", + fields: fields{ + tablet: tablet, + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &TabletVStreamerClient{ + tablet: tcase.fields.tablet, + } + + err := vsClient.Open(tcase.args.ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + err = vsClient.Close(tcase.args.ctx) + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Close() error:\n%v, want\n%v", err, tcase.err) + } + + if vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Close() isOpen set to true, expected false") + } + }) + } +} + +func TestTabletVStreamerClientVStream(t *testing.T) { + tablet := addTablet(100) + defer deleteTablet(tablet) + + vsClient := &TabletVStreamerClient{ + tablet: tablet, + target: &querypb.Target{ + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + TabletType: tablet.Type, + }, + } + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + eventsChan := make(chan *binlogdatapb.VEvent, 1000) + send := func(events []*binlogdatapb.VEvent) error { + for _, e := range events { + eventsChan <- e + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + ctx := context.Background() + err := vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + pos := masterPosition(t) + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStream(ctx, pos, filter, send) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case got := <-eventsChan: + if got.Type != binlogdatapb.VEventType_BEGIN { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_BEGIN, got.Type) + } + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestTabletVStreamerClientVStreamRows(t *testing.T) { + tablet := addTablet(100) + defer deleteTablet(tablet) + + vsClient := &TabletVStreamerClient{ + tablet: tablet, + } + + eventsChan := make(chan *querypb.Row, 1000) + send := func(streamerResponse *binlogdatapb.VStreamRowsResponse) error { + for _, row := range streamerResponse.Rows { + eventsChan <- row + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + ctx := context.Background() + err = vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStreamRows(ctx, "select * from t1", nil, send) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case <-eventsChan: + // Success got expected + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestNewMySQLVStreamerClient(t *testing.T) { + tests := []struct { + name string + want *MySQLVStreamerClient + }{ + { + name: "sets conn params for MySQLVStreamerClient ", + want: &MySQLVStreamerClient{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + }, + }, + } + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + if got := NewMySQLVStreamerClient(); !reflect.DeepEqual(got, tcase.want) { + t.Errorf("NewMySQLVStreamerClient() = %v, want %v", got, tcase.want) + } + }) + } +} + +func TestMySQLVStreamerClientOpen(t *testing.T) { + type fields struct { + isOpen bool + sourceConnParams *mysql.ConnParams + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "initializes streamer correctly", + fields: fields{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + }, + args: args{ + ctx: context.Background(), + }, + }, + { + name: "returns error when invalid conn params are provided", + fields: fields{ + sourceConnParams: &mysql.ConnParams{ + Host: "invalidhost", + Port: 3306, + }, + }, + args: args{ + ctx: context.Background(), + }, + err: "failed: dial tcp: lookup invalidhost", + }, + } + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: tcase.fields.sourceConnParams, + } + + err := vsClient.Open(tcase.args.ctx) + + if err != nil { + if !strings.Contains(err.Error(), tcase.err) { + t.Errorf("MySQLVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + return + } + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + + if !vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Open() isOpen set to false, expected true") + } + + if !vsClient.sourceSe.IsOpen() { + t.Errorf("MySQLVStreamerClient.Open() expected sourceSe to be opened") + } + }) + } +} + +func TestMySQLVStreamerClientClose(t *testing.T) { + type fields struct { + isOpen bool + sourceConnParams *mysql.ConnParams + vsEngine *vstreamer.Engine + sourceSe *schema.Engine + } + type args struct { + ctx context.Context + } + + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "closes engine correctly", + fields: fields{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + isOpen: tcase.fields.isOpen, + sourceConnParams: tcase.fields.sourceConnParams, + } + + err := vsClient.Open(tcase.args.ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + err = vsClient.Close(tcase.args.ctx) + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Close() error:\n%v, want\n%v", err, tcase.err) + } + + if vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Close() isOpen set to true, expected false") + } + + if vsClient.sourceSe.IsOpen() { + t.Errorf("MySQLVStreamerClient.Close() expected sourceSe to be closed") + } + }) + } +} + +func TestMySQLVStreamerClientVStream(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + } + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + eventsChan := make(chan *binlogdatapb.VEvent, 1000) + send := func(events []*binlogdatapb.VEvent) error { + for _, e := range events { + eventsChan <- e + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + ctx := context.Background() + err := vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + pos := masterPosition(t) + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStream(ctx, pos, filter, send) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case got := <-eventsChan: + if got.Type != binlogdatapb.VEventType_BEGIN { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_BEGIN, got.Type) + } + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestMySQLVStreamerClientVStreamRows(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + } + + eventsChan := make(chan *querypb.Row, 1000) + send := func(streamerResponse *binlogdatapb.VStreamRowsResponse) error { + for _, row := range streamerResponse.Rows { + eventsChan <- row + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + + ctx := context.Background() + err = vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStreamRows(ctx, "select * from t1", nil, send) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case <-eventsChan: + // Success got expected + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 0e7ec722e62..2aef69cbed6 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -54,7 +54,7 @@ func TestStrictMode(t *testing.T) { // config.EnforceStrictTransTable is true by default. qe := NewQueryEngine(DummyChecker, schema.NewEngine(DummyChecker, config), config) qe.InitDBConfig(dbcfgs) - qe.se.InitDBConfig(dbcfgs) + qe.se.InitDBConfig(dbcfgs.DbaWithDB()) qe.se.Open() if err := qe.Open(); err != nil { t.Error(err) @@ -298,7 +298,7 @@ func newTestQueryEngine(queryPlanCacheSize int, idleTimeout time.Duration, stric config.IdleTimeout = float64(idleTimeout) / 1e9 se := schema.NewEngine(DummyChecker, config) qe := NewQueryEngine(DummyChecker, se, config) - se.InitDBConfig(dbcfgs) + se.InitDBConfig(dbcfgs.DbaWithDB()) qe.InitDBConfig(dbcfgs) return qe } diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 202495e231c..a10e36473cd 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -31,7 +31,6 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/concurrency" - "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -48,7 +47,7 @@ type notifier func(full map[string]*Table, created, altered, dropped []string) // Engine stores the schema info and performs operations that // keep itself up-to-date. type Engine struct { - dbconfigs *dbconfigs.DBConfigs + cp *mysql.ConnParams // mu protects the following fields. mu sync.Mutex @@ -100,8 +99,8 @@ func NewEngine(checker connpool.MySQLChecker, config tabletenv.TabletConfig) *En } // InitDBConfig must be called before Open. -func (se *Engine) InitDBConfig(dbcfgs *dbconfigs.DBConfigs) { - se.dbconfigs = dbcfgs +func (se *Engine) InitDBConfig(cp *mysql.ConnParams) { + se.cp = cp } // Open initializes the Engine. Calling Open on an already @@ -115,8 +114,7 @@ func (se *Engine) Open() error { start := time.Now() defer func() { log.Infof("Time taken to load the schema: %v", time.Since(start)) }() ctx := tabletenv.LocalContext() - dbaParams := se.dbconfigs.DbaWithDB() - se.conns.Open(dbaParams, dbaParams, dbaParams) + se.conns.Open(se.cp, se.cp, se.cp) conn, err := se.conns.Get(ctx) if err != nil { @@ -195,6 +193,13 @@ func (se *Engine) Open() error { return nil } +// IsOpen() checks if engine is open +func (se *Engine) IsOpen() bool { + se.mu.Lock() + defer se.mu.Unlock() + return se.isOpen +} + // Close shuts down Engine and is idempotent. // It can be re-opened after Close. func (se *Engine) Close() { @@ -306,6 +311,17 @@ func (se *Engine) Reload(ctx context.Context) error { return rec.Error() } +// LoadTableBasic loads a table with minimal info. This is used by vstreamer +// to load _vt.resharding_journal. +func (se *Engine) LoadTableBasic(ctx context.Context, tableName string) (*Table, error) { + conn, err := se.conns.Get(ctx) + if err != nil { + return nil, err + } + defer conn.Recycle() + return LoadTableBasic(conn, tableName) +} + func (se *Engine) mysqlTime(ctx context.Context, conn *connpool.DBConn) (int64, error) { tm, err := conn.Exec(ctx, "select unix_timestamp()", 1, false) if err != nil { diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 259140a84f6..a25a4dac0a7 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -412,7 +412,7 @@ func newEngine(queryPlanCacheSize int, reloadTime time.Duration, idleTimeout tim config.SchemaReloadTime = float64(reloadTime) / 1e9 config.IdleTimeout = float64(idleTimeout) / 1e9 se := NewEngine(DummyChecker, config) - se.InitDBConfig(newDBConfigs(db)) + se.InitDBConfig(newDBConfigs(db).DbaWithDB()) return se } diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go index 273b41a89db..b530dcb09da 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table.go +++ b/go/vt/vttablet/tabletserver/schema/load_table.go @@ -54,6 +54,15 @@ func LoadTable(conn *connpool.DBConn, tableName string, tableType string, commen return ta, nil } +// LoadTableBaisc creates a Table with just the column info loaded. +func LoadTableBasic(conn *connpool.DBConn, tableName string) (*Table, error) { + ta := NewTable(tableName) + if err := fetchColumns(ta, conn, tableName); err != nil { + return nil, err + } + return ta, nil +} + func fetchColumns(ta *Table, conn *connpool.DBConn, sqlTableName string) error { qr, err := conn.Exec(tabletenv.LocalContext(), fmt.Sprintf("select * from %s where 1 != 1", sqlTableName), 0, true) if err != nil { diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 1de6dbfa8dd..8048a2cebcd 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -386,14 +386,14 @@ func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.D tsv.target = target tsv.dbconfigs = dbcfgs - tsv.se.InitDBConfig(tsv.dbconfigs) + tsv.se.InitDBConfig(tsv.dbconfigs.DbaWithDB()) tsv.qe.InitDBConfig(tsv.dbconfigs) tsv.teCtrl.InitDBConfig(tsv.dbconfigs) tsv.hw.InitDBConfig(tsv.dbconfigs) tsv.hr.InitDBConfig(tsv.dbconfigs) tsv.messager.InitDBConfig(tsv.dbconfigs) tsv.watcher.InitDBConfig(tsv.dbconfigs) - tsv.vstreamer.InitDBConfig(tsv.dbconfigs) + tsv.vstreamer.InitDBConfig(tsv.dbconfigs.DbaWithDB()) return nil } diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index b8fd1a3d204..6e385dcf79f 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -962,7 +962,7 @@ func TestTabletServerBeginFail(t *testing.T) { defer cancel() tsv.Begin(ctx, &target, nil) _, err = tsv.Begin(ctx, &target, nil) - want := "transaction pool connection limit exceeded" + want := "transaction pool aborting request due to already expired context" if err == nil || err.Error() != want { t.Fatalf("Begin err: %v, want %v", err, want) } diff --git a/go/vt/vttablet/tabletserver/tx_pool.go b/go/vt/vttablet/tabletserver/tx_pool.go index e7ecfcc513d..42de72c3f0d 100644 --- a/go/vt/vttablet/tabletserver/tx_pool.go +++ b/go/vt/vttablet/tabletserver/tx_pool.go @@ -245,6 +245,9 @@ func (axp *TxPool) Begin(ctx context.Context, options *querypb.ExecuteOptions) ( switch err { case connpool.ErrConnPoolClosed: return 0, "", err + case pools.ErrCtxTimeout: + axp.LogActive() + return 0, "", vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED, "transaction pool aborting request due to already expired context") case pools.ErrTimeout: axp.LogActive() return 0, "", vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED, "transaction pool connection limit exceeded") diff --git a/go/vt/vttablet/tabletserver/tx_pool_test.go b/go/vt/vttablet/tabletserver/tx_pool_test.go index cdd11cac5a0..eccb7030064 100644 --- a/go/vt/vttablet/tabletserver/tx_pool_test.go +++ b/go/vt/vttablet/tabletserver/tx_pool_test.go @@ -470,6 +470,25 @@ func TestTxPoolBeginWithError(t *testing.T) { } } +func TestTxPoolCancelledContextError(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + db.AddRejectedQuery("begin", errRejected) + txPool := newTxPool() + txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + defer txPool.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}) + want := "transaction pool aborting request due to already expired context" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("Unexpected error: %v, want %s", err, want) + } + if got, want := vterrors.Code(err), vtrpcpb.Code_RESOURCE_EXHAUSTED; got != want { + t.Errorf("wrong error code error: got = %v, want = %v", got, want) + } +} + func TestTxPoolRollbackFail(t *testing.T) { sql := "alter table test_table add test_column int" db := fakesqldb.New(t) diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 67ae75be8c0..19b54c572d6 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -28,7 +28,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" - "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo" @@ -97,8 +96,8 @@ func NewEngine(ts srvtopo.Server, se *schema.Engine) *Engine { } // InitDBConfig performs saves the required info from dbconfigs for future use. -func (vse *Engine) InitDBConfig(dbcfgs *dbconfigs.DBConfigs) { - vse.cp = dbcfgs.DbaWithDB() +func (vse *Engine) InitDBConfig(cp *mysql.ConnParams) { + vse.cp = cp } // Open starts the Engine service. @@ -114,6 +113,13 @@ func (vse *Engine) Open(keyspace, cell string) error { return nil } +// IsOpen checks if the engine is opened +func (vse *Engine) IsOpen() bool { + vse.mu.Lock() + defer vse.mu.Unlock() + return vse.isOpen +} + // Close closes the Engine service. func (vse *Engine) Close() { func() { @@ -160,7 +166,7 @@ func (vse *Engine) Stream(ctx context.Context, startPos string, filter *binlogda if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - streamer := newVStreamer(ctx, vse.cp, vse.se, startPos, filter, vse.kschema, send) + streamer := NewVStreamer(ctx, vse.cp, vse.se, startPos, filter, vse.kschema, send) idx := vse.streamIdx vse.streamers[idx] = streamer vse.streamIdx++ @@ -200,7 +206,7 @@ func (vse *Engine) StreamRows(ctx context.Context, query string, lastpk []sqltyp if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - rowStreamer := newRowStreamer(ctx, vse.cp, vse.se, query, lastpk, vse.kschema, send) + rowStreamer := NewRowStreamer(ctx, vse.cp, vse.se, query, lastpk, vse.kschema, send) idx := vse.streamIdx vse.rowStreamers[idx] = rowStreamer vse.streamIdx++ diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go index 575b146b6e0..28c5da5643d 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go @@ -26,7 +26,8 @@ import ( binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) -var shardedVSchema = `{ +var ( + shardedVSchema = `{ "sharded": true, "vindexes": { "hash": { @@ -45,6 +46,32 @@ var shardedVSchema = `{ } }` + multicolumnVSchema = `{ + "sharded": true, + "vindexes": { + "region_vdx": { + "type": "region_experimental", + "params": { + "region_bytes": "1" + } + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "columns": [ + "region", + "id" + ], + "name": "region_vdx" + } + ] + } + } +}` +) + func TestUpdateVSchema(t *testing.T) { if testing.Short() { t.Skip() diff --git a/go/vt/vttablet/tabletserver/vstreamer/main_test.go b/go/vt/vttablet/tabletserver/vstreamer/main_test.go index 0b5acf7017b..cda8ad22251 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/main_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/main_test.go @@ -49,7 +49,7 @@ func TestMain(m *testing.M) { // engine cannot be initialized in testenv because it introduces // circular dependencies. engine = NewEngine(env.SrvTopo, env.SchemaEngine) - engine.InitDBConfig(env.Dbcfgs) + engine.InitDBConfig(env.Dbcfgs.DbaWithDB()) engine.Open(env.KeyspaceName, env.Cells[0]) defer engine.Close() diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 43affd74d35..8e8f211cde3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -35,18 +35,28 @@ import ( // Plan represents the plan for a table. type Plan struct { - Table *Table - ColExprs []ColExpr - VindexColumn int - Vindex vindexes.Vindex - KeyRange *topodatapb.KeyRange + Table *Table + ColExprs []ColExpr + + // Vindex, VindexColumns and KeyRange, if set, will be used + // to filter the row. + Vindex vindexes.Vindex + VindexColumns []int + KeyRange *topodatapb.KeyRange } // ColExpr represents a column expression. type ColExpr struct { + // ColNum specifies the source column value. ColNum int - Alias sqlparser.ColIdent - Type querypb.Type + + // Vindex and VindexColumns, if set, will be used to generate + // a keyspace_id. If so, ColNum is ignored. + Vindex vindexes.Vindex + VindexColumns []int + + Alias sqlparser.ColIdent + Type querypb.Type } // Table contains the metadata for a table. @@ -70,33 +80,62 @@ func (plan *Plan) fields() []*querypb.Field { // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { + if plan.Vindex != nil { + vindexValues := make([]sqltypes.Value, 0, len(plan.VindexColumns)) + for _, col := range plan.VindexColumns { + vindexValues = append(vindexValues, values[col]) + } + ksid, err := getKeyspaceID(vindexValues, plan.Vindex) + if err != nil { + return false, nil, err + } + if !key.KeyRangeContains(plan.KeyRange, ksid) { + return false, nil, nil + } + } + result := make([]sqltypes.Value, len(plan.ColExprs)) for i, colExpr := range plan.ColExprs { if colExpr.ColNum >= len(values) { return false, nil, fmt.Errorf("index out of range, colExpr.ColNum: %d, len(values): %d", colExpr.ColNum, len(values)) } - result[i] = values[colExpr.ColNum] - } - if plan.Vindex == nil { - return true, result, nil + if colExpr.Vindex == nil { + result[i] = values[colExpr.ColNum] + } else { + vindexValues := make([]sqltypes.Value, 0, len(colExpr.VindexColumns)) + for _, col := range colExpr.VindexColumns { + vindexValues = append(vindexValues, values[col]) + } + ksid, err := getKeyspaceID(vindexValues, colExpr.Vindex) + if err != nil { + return false, nil, err + } + result[i] = sqltypes.MakeTrusted(sqltypes.VarBinary, []byte(ksid)) + } } + return true, result, nil +} - // Filter by Vindex. - destinations, err := plan.Vindex.Map(nil, []sqltypes.Value{result[plan.VindexColumn]}) +func getKeyspaceID(values []sqltypes.Value, vindex vindexes.Vindex) (key.DestinationKeyspaceID, error) { + destinations, err := vindexes.Map(vindex, nil, [][]sqltypes.Value{values}) if err != nil { - return false, nil, err + return nil, err } if len(destinations) != 1 { - return false, nil, fmt.Errorf("mapping row to keyspace id returned an invalid array of destinations: %v", key.DestinationsString(destinations)) + return nil, fmt.Errorf("mapping row to keyspace id returned an invalid array of destinations: %v", key.DestinationsString(destinations)) } ksid, ok := destinations[0].(key.DestinationKeyspaceID) if !ok || len(ksid) == 0 { - return false, nil, fmt.Errorf("could not map %v to a keyspace id, got destination %v", result[plan.VindexColumn], destinations[0]) + return nil, fmt.Errorf("could not map %v to a keyspace id, got destination %v", values, destinations[0]) } - if !key.KeyRangeContains(plan.KeyRange, ksid) { - return false, nil, nil + return ksid, nil +} + +func mustSendStmt(query mysql.Query, dbname string) bool { + if query.Database != "" && query.Database != dbname { + return false } - return true, result, nil + return true } func mustSendDDL(query mysql.Query, dbname string, filter *binlogdatapb.Filter) bool { @@ -200,14 +239,12 @@ func buildREPlan(ti *Table, kschema *vindexes.KeyspaceSchema, filter string) (*P if len(table.ColumnVindexes) == 0 { return nil, fmt.Errorf("table %s has no primary vindex", ti.Name) } - // findColumn can be used here because result column list is same - // as source. - colnum, err := findColumn(ti, table.ColumnVindexes[0].Columns[0]) + plan.Vindex = table.ColumnVindexes[0].Vindex + var err error + plan.VindexColumns, err = buildVindexColumns(plan.Table, table.ColumnVindexes[0].Columns) if err != nil { return nil, err } - plan.VindexColumn = colnum - plan.Vindex = table.ColumnVindexes[0].Vindex // Parse keyrange. keyranges, err := key.ParseShardingSpec(filter) @@ -233,7 +270,7 @@ func buildTablePlan(ti *Table, kschema *vindexes.KeyspaceSchema, query string) ( plan := &Plan{ Table: ti, } - if err := plan.analyzeExprs(sel.SelectExprs); err != nil { + if err := plan.analyzeExprs(kschema, sel.SelectExprs); err != nil { return nil, err } @@ -277,10 +314,10 @@ func analyzeSelect(query string) (sel *sqlparser.Select, fromTable sqlparser.Tab return sel, fromTable, nil } -func (plan *Plan) analyzeExprs(selExprs sqlparser.SelectExprs) error { +func (plan *Plan) analyzeExprs(kschema *vindexes.KeyspaceSchema, selExprs sqlparser.SelectExprs) error { if _, ok := selExprs[0].(*sqlparser.StarExpr); !ok { for _, expr := range selExprs { - cExpr, err := plan.analyzeExpr(expr) + cExpr, err := plan.analyzeExpr(kschema, expr) if err != nil { return err } @@ -300,34 +337,64 @@ func (plan *Plan) analyzeExprs(selExprs sqlparser.SelectExprs) error { return nil } -func (plan *Plan) analyzeExpr(selExpr sqlparser.SelectExpr) (cExpr ColExpr, err error) { +func (plan *Plan) analyzeExpr(kschema *vindexes.KeyspaceSchema, selExpr sqlparser.SelectExpr) (cExpr ColExpr, err error) { aliased, ok := selExpr.(*sqlparser.AliasedExpr) if !ok { return ColExpr{}, fmt.Errorf("unsupported: %v", sqlparser.String(selExpr)) } - as := aliased.As - if as.IsEmpty() { - as = sqlparser.NewColIdent(sqlparser.String(aliased.Expr)) - } - colname, ok := aliased.Expr.(*sqlparser.ColName) - if !ok { + switch inner := aliased.Expr.(type) { + case *sqlparser.ColName: + if !inner.Qualifier.IsEmpty() { + return ColExpr{}, fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(inner)) + } + colnum, err := findColumn(plan.Table, inner.Name) + if err != nil { + return ColExpr{}, err + } + as := aliased.As + if as.IsEmpty() { + as = sqlparser.NewColIdent(sqlparser.String(aliased.Expr)) + } + return ColExpr{ + ColNum: colnum, + Alias: as, + Type: plan.Table.Columns[colnum].Type, + }, nil + case *sqlparser.FuncExpr: + if inner.Name.Lowered() != "keyspace_id" { + return ColExpr{}, fmt.Errorf("unsupported function: %v", sqlparser.String(inner)) + } + if len(inner.Exprs) != 0 { + return ColExpr{}, fmt.Errorf("unexpected: %v", sqlparser.String(inner)) + } + table := kschema.Tables[plan.Table.Name] + if table == nil { + return ColExpr{}, fmt.Errorf("no vschema definition for table %s", plan.Table.Name) + } + // Get Primary Vindex. + if len(table.ColumnVindexes) == 0 { + return ColExpr{}, fmt.Errorf("table %s has no primary vindex", plan.Table.Name) + } + vindexColumns, err := buildVindexColumns(plan.Table, table.ColumnVindexes[0].Columns) + if err != nil { + return ColExpr{}, err + } + return ColExpr{ + Vindex: table.ColumnVindexes[0].Vindex, + VindexColumns: vindexColumns, + Alias: sqlparser.NewColIdent("keyspace_id"), + Type: sqltypes.VarBinary, + }, nil + default: return ColExpr{}, fmt.Errorf("unsupported: %v", sqlparser.String(aliased.Expr)) } - if !colname.Qualifier.IsEmpty() { - return ColExpr{}, fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(colname)) - } - colnum, err := findColumn(plan.Table, colname.Name) - if err != nil { - return ColExpr{}, err - } - return ColExpr{ColNum: colnum, Alias: as, Type: plan.Table.Columns[colnum].Type}, nil } func (plan *Plan) analyzeInKeyRange(kschema *vindexes.KeyspaceSchema, exprs sqlparser.SelectExprs) error { - var colname sqlparser.ColIdent + var colnames []sqlparser.ColIdent var krExpr sqlparser.SelectExpr - switch len(exprs) { - case 1: + switch { + case len(exprs) == 1: table := kschema.Tables[plan.Table.Name] if table == nil { return fmt.Errorf("no vschema definition for table %s", plan.Table.Name) @@ -336,23 +403,26 @@ func (plan *Plan) analyzeInKeyRange(kschema *vindexes.KeyspaceSchema, exprs sqlp if len(table.ColumnVindexes) == 0 { return fmt.Errorf("table %s has no primary vindex", plan.Table.Name) } - colname = table.ColumnVindexes[0].Columns[0] + colnames = table.ColumnVindexes[0].Columns plan.Vindex = table.ColumnVindexes[0].Vindex krExpr = exprs[0] - case 3: - aexpr, ok := exprs[0].(*sqlparser.AliasedExpr) - if !ok { - return fmt.Errorf("unexpected: %v", sqlparser.String(exprs[0])) - } - qualifiedName, ok := aexpr.Expr.(*sqlparser.ColName) - if !ok { - return fmt.Errorf("unexpected: %v", sqlparser.String(exprs[0])) - } - if !qualifiedName.Qualifier.IsEmpty() { - return fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(colname)) + case len(exprs) >= 3: + for _, expr := range exprs[:len(exprs)-2] { + aexpr, ok := expr.(*sqlparser.AliasedExpr) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + qualifiedName, ok := aexpr.Expr.(*sqlparser.ColName) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if !qualifiedName.Qualifier.IsEmpty() { + return fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(qualifiedName)) + } + colnames = append(colnames, qualifiedName.Name) } - colname = qualifiedName.Name - vtype, err := selString(exprs[1]) + + vtype, err := selString(exprs[len(exprs)-2]) if err != nil { return err } @@ -363,20 +433,15 @@ func (plan *Plan) analyzeInKeyRange(kschema *vindexes.KeyspaceSchema, exprs sqlp if !plan.Vindex.IsUnique() { return fmt.Errorf("vindex must be Unique to be used for VReplication: %s", vtype) } - krExpr = exprs[2] + + krExpr = exprs[len(exprs)-1] default: return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(exprs)) } - found := false - for i, cExpr := range plan.ColExprs { - if cExpr.Alias.Equal(colname) { - found = true - plan.VindexColumn = i - break - } - } - if !found { - return fmt.Errorf("keyrange expression does not reference a column in the select list: %v", sqlparser.String(colname)) + var err error + plan.VindexColumns, err = buildVindexColumns(plan.Table, colnames) + if err != nil { + return err } kr, err := selString(krExpr) if err != nil { @@ -405,6 +470,18 @@ func selString(expr sqlparser.SelectExpr) (string, error) { return string(val.Val), nil } +func buildVindexColumns(ti *Table, colnames []sqlparser.ColIdent) ([]int, error) { + vindexColumns := make([]int, 0, len(colnames)) + for _, colname := range colnames { + colnum, err := findColumn(ti, colname) + if err != nil { + return nil, err + } + vindexColumns = append(vindexColumns, colnum) + } + return vindexColumns, nil +} + func findColumn(ti *Table, name sqlparser.ColIdent) (int, error) { for i, col := range ti.Columns { if name.Equal(col.Name) { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 731de846b9c..e6d38059e94 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -41,8 +41,11 @@ func init() { "hash": { "type": "hash" }, - "lookup": { - "type": "lookup" + "region_vdx": { + "type": "region_experimental", + "params": { + "region_bytes": "1" + } } }, "tables": { @@ -53,6 +56,17 @@ func init() { "name": "hash" } ] + }, + "regional": { + "column_vindexes": [ + { + "columns": [ + "region", + "id" + ], + "name": "region_vdx" + } + ] } } }` @@ -177,6 +191,19 @@ func TestPlanbuilder(t *testing.T) { Type: sqltypes.VarBinary, }}, } + regional := &Table{ + Name: "regional", + Columns: []schema.TableColumn{{ + Name: sqlparser.NewColIdent("region"), + Type: sqltypes.Int64, + }, { + Name: sqlparser.NewColIdent("id"), + Type: sqltypes.Int64, + }, { + Name: sqlparser.NewColIdent("val"), + Type: sqltypes.VarBinary, + }}, + } testcases := []struct { inTable *Table @@ -210,7 +237,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - VindexColumn: 0, + VindexColumns: []int{0}, }, }, { inTable: t1, @@ -267,7 +294,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumn: 1, + VindexColumns: []int{0}, }, }, { inTable: t1, @@ -282,11 +309,41 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumn: 1, + VindexColumns: []int{0}, }, }, { inTable: t2, inRule: &binlogdatapb.Rule{Match: "/t1/"}, + }, { + inTable: regional, + inRule: &binlogdatapb.Rule{Match: "regional", Filter: "select val, id from regional where in_keyrange('-80')"}, + outPlan: &Plan{ + ColExprs: []ColExpr{{ + ColNum: 2, + Alias: sqlparser.NewColIdent("val"), + Type: sqltypes.VarBinary, + }, { + ColNum: 1, + Alias: sqlparser.NewColIdent("id"), + Type: sqltypes.Int64, + }}, + VindexColumns: []int{0, 1}, + }, + }, { + inTable: regional, + inRule: &binlogdatapb.Rule{Match: "regional", Filter: "select id, keyspace_id() from regional"}, + outPlan: &Plan{ + ColExprs: []ColExpr{{ + ColNum: 1, + Alias: sqlparser.NewColIdent("id"), + Type: sqltypes.Int64, + }, { + Alias: sqlparser.NewColIdent("keyspace_id"), + Vindex: testKSChema.Vindexes["region_vdx"], + VindexColumns: []int{0, 1}, + Type: sqltypes.VarBinary, + }}, + }, }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "/*/"}, @@ -355,10 +412,6 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(1, 'hash', '-80')"}, outErr: `unexpected: 1`, - }, { - inTable: t1, - inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(none, 'hash', '-80')"}, - outErr: `keyrange expression does not reference a column in the select list: none`, }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(id, 'lookup', '-80')"}, @@ -383,7 +436,7 @@ func TestPlanbuilder(t *testing.T) { }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val, max(val) from t1"}, - outErr: `unsupported: max(val)`, + outErr: `unsupported function: max(val)`, }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id+1, val from t1"}, diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index c4114d451c0..3bdf0f2bdab 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -48,7 +48,7 @@ type rowStreamer struct { sendQuery string } -func newRowStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, query string, lastpk []sqltypes.Value, kschema *vindexes.KeyspaceSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { +func NewRowStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, query string, lastpk []sqltypes.Value, kschema *vindexes.KeyspaceSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { ctx, cancel := context.WithCancel(ctx) return &rowStreamer{ ctx: ctx, diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index be2eb3070fe..84b2365a75e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -92,7 +92,7 @@ func Init() (*Env, error) { }, }, }, - ExtraMyCnf: []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}, + ExtraMyCnf: []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")}, OnlyMySQL: true, } te.cluster = &vttest.LocalCluster{ @@ -106,7 +106,7 @@ func Init() (*Env, error) { te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) - te.SchemaEngine.InitDBConfig(te.Dbcfgs) + te.SchemaEngine.InitDBConfig(te.Dbcfgs.DbaWithDB()) // The first vschema should not be empty. Leads to Node not found error. // TODO(sougou): need to fix the bug. diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 791ee38e757..09c3eb3967e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -23,6 +23,7 @@ import ( "io" "time" + "github.com/golang/protobuf/proto" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog" @@ -53,9 +54,10 @@ type vstreamer struct { send func([]*binlogdatapb.VEvent) error // A kschema is a VSchema for just one keyspace. - kevents chan *vindexes.KeyspaceSchema - kschema *vindexes.KeyspaceSchema - plans map[uint64]*streamerPlan + kevents chan *vindexes.KeyspaceSchema + kschema *vindexes.KeyspaceSchema + plans map[uint64]*streamerPlan + journalTableID uint64 // format and pos are updated by parseEvent. format mysql.BinlogFormat @@ -69,7 +71,7 @@ type streamerPlan struct { TableMap *mysql.TableMap } -func newVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { +func NewVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { ctx, cancel := context.WithCancel(ctx) return &vstreamer{ ctx: ctx, @@ -142,16 +144,26 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog // If a single row exceeds the packet size, it will be in its own packet. bufferAndTransmit := func(vevent *binlogdatapb.VEvent) error { switch vevent.Type { - case binlogdatapb.VEventType_GTID, binlogdatapb.VEventType_BEGIN, binlogdatapb.VEventType_FIELD: - // We never have to send GTID, BEGIN or FIELD events on their own. + case binlogdatapb.VEventType_GTID, binlogdatapb.VEventType_BEGIN, binlogdatapb.VEventType_FIELD, binlogdatapb.VEventType_JOURNAL: + // We never have to send GTID, BEGIN, FIELD events on their own. bufferedEvents = append(bufferedEvents, vevent) - case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_DDL, binlogdatapb.VEventType_HEARTBEAT: - // COMMIT, DDL and HEARTBEAT must be immediately sent. + case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_DDL, binlogdatapb.VEventType_OTHER, binlogdatapb.VEventType_HEARTBEAT: + // COMMIT, DDL, OTHER and HEARTBEAT must be immediately sent. bufferedEvents = append(bufferedEvents, vevent) vevents := bufferedEvents bufferedEvents = nil curSize = 0 return vs.send(vevents) + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE, binlogdatapb.VEventType_REPLACE: + newSize := len(vevent.GetDml()) + if curSize+newSize > *PacketSize { + vevents := bufferedEvents + bufferedEvents = []*binlogdatapb.VEvent{vevent} + curSize = newSize + return vs.send(vevents) + } + curSize += newSize + bufferedEvents = append(bufferedEvents, vevent) case binlogdatapb.VEventType_ROW: // ROW events happen inside transactions. So, we can chunk them. // Buffer everything until packet size is reached, and then send. @@ -281,12 +293,11 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e }) } vs.pos = mysql.AppendGTID(vs.pos, gtid) + case ev.IsXID(): vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_GTID, Gtid: mysql.EncodePosition(vs.pos), - }) - case ev.IsXID(): - vevents = append(vevents, &binlogdatapb.VEvent{ + }, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_COMMIT, }) case ev.IsQuery(): @@ -294,7 +305,41 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if err != nil { return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } + // Insert/Delete/Update are supported only to be used in the context of external mysql streams where source databases + // could be using SBR. Vitess itself will never run into cases where it needs to consume non rbr statements. switch cat := sqlparser.Preview(q.SQL); cat { + case sqlparser.StmtInsert: + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_INSERT, + Dml: q.SQL, + }) + } + case sqlparser.StmtUpdate: + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_UPDATE, + Dml: q.SQL, + }) + } + case sqlparser.StmtDelete: + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_DELETE, + Dml: q.SQL, + }) + } + case sqlparser.StmtReplace: + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_REPLACE, + Dml: q.SQL, + }) + } case sqlparser.StmtBegin: vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_BEGIN, @@ -306,17 +351,20 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e case sqlparser.StmtDDL: if mustSendDDL(q, vs.cp.DbName, vs.filter) { vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_GTID, + Gtid: mysql.EncodePosition(vs.pos), + }, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_DDL, Ddl: q.SQL, }) } else { - vevents = append(vevents, - &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_BEGIN, - }, - &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_COMMIT, - }) + // If the DDL need not be sent, send a dummy OTHER event. + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_GTID, + Gtid: mysql.EncodePosition(vs.pos), + }, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_OTHER, + }) } // Proactively reload schema. // If the DDL adds a column, comparing with an older snapshot of the @@ -324,88 +372,39 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e vs.se.Reload(vs.ctx) case sqlparser.StmtOther: // These are DBA statements like REPAIR that can be ignored. + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_GTID, + Gtid: mysql.EncodePosition(vs.pos), + }, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_OTHER, + }) default: return nil, fmt.Errorf("unexpected statement type %s in row-based replication: %q", cat, q.SQL) } case ev.IsTableMap(): // This is very frequent. It precedes every row event. id := ev.TableID(vs.format) + if _, ok := vs.plans[id]; ok { + return nil, nil + } tm, err := ev.TableMap(vs.format) if err != nil { return nil, err } - // We have to build a plan only for new ids. - if _, ok := vs.plans[id]; ok { - return nil, nil + if tm.Database == "_vt" && tm.Name == "resharding_journal" { + return nil, vs.buildJournalPlan(id, tm) } if tm.Database != "" && tm.Database != vs.cp.DbName { vs.plans[id] = nil return nil, nil } - tableName := tm.Name - var cols []schema.TableColumn - for i, typ := range tm.Types { - t, err := sqltypes.MySQLToType(int64(typ), 0) - if err != nil { - return nil, fmt.Errorf("unsupported type: %d, position: %d", typ, i) - } - cols = append(cols, schema.TableColumn{ - Name: sqlparser.NewColIdent(fmt.Sprintf("@%d", i+1)), - Type: t, - }) - } - st := vs.se.GetTable(sqlparser.NewTableIdent(tm.Name)) - if st == nil { - if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { - return nil, fmt.Errorf("unknown table %v in schema", tm.Name) - } - } else { - if len(st.Columns) < len(tm.Types) && vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { - return nil, fmt.Errorf("cannot determine table columns for %s: event has %d columns, current schema has %d: %#v", tm.Name, len(tm.Types), len(st.Columns), ev) - } - tableName = st.Name.String() - // check if the schema returned by schema.Engine matches with row. - schemaMatch := true - if len(tm.Types) <= len(st.Columns) { - for i := range tm.Types { - t := cols[i].Type - if !sqltypes.AreTypesEquivalent(t, st.Columns[i].Type) { - schemaMatch = false - break - } - } - } else { - schemaMatch = false - } - if schemaMatch { - // Columns should be truncated to match those in tm. - cols = st.Columns[:len(tm.Types)] - } - } - - table := &Table{ - Name: tableName, - Columns: cols, - } - plan, err := buildPlan(table, vs.kschema, vs.filter) + vevent, err := vs.buildTablePlan(id, tm) if err != nil { return nil, err } - if plan == nil { - vs.plans[id] = nil - return nil, nil - } - vs.plans[id] = &streamerPlan{ - Plan: plan, - TableMap: tm, + if vevent != nil { + vevents = append(vevents, vevent) } - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_FIELD, - FieldEvent: &binlogdatapb.FieldEvent{ - TableName: plan.Table.Name, - Fields: plan.fields(), - }, - }) case ev.IsWriteRows() || ev.IsDeleteRows() || ev.IsUpdateRows(): // The existence of before and after images can be used to // identify statememt types. It's also possible that the @@ -421,36 +420,13 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if err != nil { return nil, err } - rowChanges := make([]*binlogdatapb.RowChange, 0, len(rows.Rows)) - for _, row := range rows.Rows { - beforeOK, beforeValues, err := vs.extractRowAndFilter(plan, row.Identify, rows.IdentifyColumns, row.NullIdentifyColumns) - if err != nil { - return nil, err - } - afterOK, afterValues, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns) - if err != nil { - return nil, err - } - if !beforeOK && !afterOK { - continue - } - rowChange := &binlogdatapb.RowChange{} - if beforeOK { - rowChange.Before = sqltypes.RowToProto3(beforeValues) - } - if afterOK { - rowChange.After = sqltypes.RowToProto3(afterValues) - } - rowChanges = append(rowChanges, rowChange) + if id == vs.journalTableID { + vevents, err = vs.processJounalEvent(vevents, plan, rows) + } else { + vevents, err = vs.processRowEvent(vevents, plan, rows) } - if len(rowChanges) != 0 { - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_ROW, - RowEvent: &binlogdatapb.RowEvent{ - TableName: plan.Table.Name, - RowChanges: rowChanges, - }, - }) + if err != nil { + return nil, err } } for _, vevent := range vevents { @@ -460,6 +436,167 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e return vevents, nil } +func (vs *vstreamer) buildJournalPlan(id uint64, tm *mysql.TableMap) error { + st, err := vs.se.LoadTableBasic(vs.ctx, "_vt.resharding_journal") + if err != nil { + return err + } + if len(st.Columns) < len(tm.Types) { + return fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + } + table := &Table{ + Name: "_vt.resharding_journal", + Columns: st.Columns[:len(tm.Types)], + } + plan, err := buildREPlan(table, nil, "") + if err != nil { + return err + } + vs.plans[id] = &streamerPlan{ + Plan: plan, + TableMap: tm, + } + vs.journalTableID = id + return nil +} + +func (vs *vstreamer) buildTablePlan(id uint64, tm *mysql.TableMap) (*binlogdatapb.VEvent, error) { + cols, err := vs.buildTableColumns(id, tm) + if err != nil { + return nil, err + } + + table := &Table{ + Name: tm.Name, + Columns: cols, + } + plan, err := buildPlan(table, vs.kschema, vs.filter) + if err != nil { + return nil, err + } + if plan == nil { + vs.plans[id] = nil + return nil, nil + } + vs.plans[id] = &streamerPlan{ + Plan: plan, + TableMap: tm, + } + return &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_FIELD, + FieldEvent: &binlogdatapb.FieldEvent{ + TableName: plan.Table.Name, + Fields: plan.fields(), + }, + }, nil +} + +func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]schema.TableColumn, error) { + var cols []schema.TableColumn + for i, typ := range tm.Types { + t, err := sqltypes.MySQLToType(int64(typ), 0) + if err != nil { + return nil, fmt.Errorf("unsupported type: %d, position: %d", typ, i) + } + cols = append(cols, schema.TableColumn{ + Name: sqlparser.NewColIdent(fmt.Sprintf("@%d", i+1)), + Type: t, + }) + } + + st := vs.se.GetTable(sqlparser.NewTableIdent(tm.Name)) + if st == nil { + if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { + return nil, fmt.Errorf("unknown table %v in schema", tm.Name) + } + return cols, nil + } + + if len(st.Columns) < len(tm.Types) { + if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { + return nil, fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + } + return cols, nil + } + + // check if the schema returned by schema.Engine matches with row. + for i := range tm.Types { + if !sqltypes.AreTypesEquivalent(cols[i].Type, st.Columns[i].Type) { + return cols, nil + } + } + + // Columns should be truncated to match those in tm. + cols = st.Columns[:len(tm.Types)] + return cols, nil +} + +func (vs *vstreamer) processJounalEvent(vevents []*binlogdatapb.VEvent, plan *streamerPlan, rows mysql.Rows) ([]*binlogdatapb.VEvent, error) { +nextrow: + for _, row := range rows.Rows { + afterOK, afterValues, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns) + if err != nil { + return nil, err + } + if !afterOK { + continue + } + for i, fld := range plan.fields() { + switch fld.Name { + case "db_name": + if afterValues[i].ToString() != vs.cp.DbName { + continue nextrow + } + case "val": + journal := &binlogdatapb.Journal{} + if err := proto.UnmarshalText(afterValues[i].ToString(), journal); err != nil { + return nil, err + } + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_JOURNAL, + Journal: journal, + }) + } + } + } + return vevents, nil +} + +func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *streamerPlan, rows mysql.Rows) ([]*binlogdatapb.VEvent, error) { + rowChanges := make([]*binlogdatapb.RowChange, 0, len(rows.Rows)) + for _, row := range rows.Rows { + beforeOK, beforeValues, err := vs.extractRowAndFilter(plan, row.Identify, rows.IdentifyColumns, row.NullIdentifyColumns) + if err != nil { + return nil, err + } + afterOK, afterValues, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns) + if err != nil { + return nil, err + } + if !beforeOK && !afterOK { + continue + } + rowChange := &binlogdatapb.RowChange{} + if beforeOK { + rowChange.Before = sqltypes.RowToProto3(beforeValues) + } + if afterOK { + rowChange.After = sqltypes.RowToProto3(afterValues) + } + rowChanges = append(rowChanges, rowChange) + } + if len(rowChanges) != 0 { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_ROW, + RowEvent: &binlogdatapb.RowEvent{ + TableName: plan.Table.Name, + RowChanges: rowChanges, + }, + }) + } + return vevents, nil +} + func (vs *vstreamer) rebuildPlans() error { for id, plan := range vs.plans { if plan == nil { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index efe84b69573..a3eca41a71f 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -58,11 +58,11 @@ func TestStatements(t *testing.T) { // MySQL issues GTID->BEGIN. // MariaDB issues BEGIN->GTID. output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, `type:ROW row_event: after: > > `, + `gtid`, `commit`, }}, }, { @@ -90,8 +90,7 @@ func TestStatements(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, `type:FIELD field_event: fields: > `, @@ -102,6 +101,7 @@ func TestStatements(t *testing.T) { `type:ROW row_event: > ` + `row_changes: > > `, + `gtid`, `commit`, }}, }, { @@ -114,23 +114,38 @@ func TestStatements(t *testing.T) { }, { // repair, optimize and analyze show up in binlog stream, but ignored by vitess. input: "repair table stream2", + output: [][]string{{ + `gtid`, + `type:OTHER `, + }}, }, { input: "optimize table stream2", + output: [][]string{{ + `gtid`, + `type:OTHER `, + }}, }, { input: "analyze table stream2", + output: [][]string{{ + `gtid`, + `type:OTHER `, + }}, }, { - // select, set, show, analyze and describe don't get logged. + // select, set, show and describe don't get logged. input: "select * from stream1", }, { input: "set @val=1", }, { input: "show tables", - }, { - input: "analyze table stream1", }, { input: "describe stream1", }} runCases(t, nil, testcases, "") + + // Test FilePos flavor + engine.cp.Flavor = "FilePos" + defer func() { engine.cp.Flavor = "" }() + runCases(t, nil, testcases, "") } func TestRegexp(t *testing.T) { @@ -164,11 +179,11 @@ func TestRegexp(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, `type:ROW row_event: after: > > `, + `gtid`, `commit`, }}, }} @@ -220,13 +235,13 @@ func TestREKeyRange(t *testing.T) { } execStatements(t, input) expectLog(ctx, t, input, ch, [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: fields: > `, `type:ROW row_event: > > `, `type:ROW row_event: after: > > `, `type:ROW row_event: > > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}) @@ -262,9 +277,122 @@ func TestREKeyRange(t *testing.T) { } execStatements(t, input) expectLog(ctx, t, input, ch, [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}) +} + +func TestInKeyRangeMultiColumn(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(region int, id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + if err := env.SetVSchema(multicolumnVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id, region, val, keyspace_id() from t1 where in_keyrange('-80')", + }}, + } + ch := startStream(ctx, t, filter, "") + + // 1, 2, 3 and 5 are in shard -80. + // 4 and 6 are in shard 80-. + input := []string{ + "begin", + "insert into t1 values (1, 1, 'aaa')", + "insert into t1 values (128, 2, 'bbb')", + // Stay in shard. + "update t1 set region = 2 where id = 1", + // Move from -80 to 80-. + "update t1 set region = 128 where id = 1", + // Move from 80- to -80. + "update t1 set region = 1 where id = 2", + "commit", + } + execStatements(t, input) + expectLog(ctx, t, input, ch, [][]string{{ + `begin`, + `type:FIELD field_event: fields: fields: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: ` + + `after: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}) +} + +func TestREMultiColumnVindex(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(region int, id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + if err := env.SetVSchema(multicolumnVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*/", + Filter: "-80", + }}, + } + ch := startStream(ctx, t, filter, "") + + // 1, 2, 3 and 5 are in shard -80. + // 4 and 6 are in shard 80-. + input := []string{ + "begin", + "insert into t1 values (1, 1, 'aaa')", + "insert into t1 values (128, 2, 'bbb')", + // Stay in shard. + "update t1 set region = 2 where id = 1", + // Move from -80 to 80-. + "update t1 set region = 128 where id = 1", + // Move from 80- to -80. + "update t1 set region = 1 where id = 2", + "commit", + } + execStatements(t, input) + expectLog(ctx, t, input, ch, [][]string{{ + `begin`, + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: after: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, `commit`, }}) } @@ -299,10 +427,10 @@ func TestSelectFilter(t *testing.T) { // MySQL issues GTID->BEGIN. // MariaDB issues BEGIN->GTID. output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }} @@ -362,12 +490,12 @@ func TestDDLAddColumn(t *testing.T) { }() expectLog(ctx, t, "ddls", ch, [][]string{{ // Current schema has 3 columns, but they'll be truncated to match the two columns in the event. - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }, { `gtid`, @@ -378,12 +506,12 @@ func TestDDLAddColumn(t *testing.T) { }, { // The plan will be updated to now include the third column // because the new table map will have three columns. - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: fields: > `, `type:ROW row_event: > > `, `type:FIELD field_event: fields: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}) } @@ -435,9 +563,8 @@ func TestUnsentDDL(t *testing.T) { }, // An unsent DDL is sent as an empty transaction. output: [][]string{{ - `gtid|begin`, - `gtid|begin`, - `commit`, + `gtid`, + `type:OTHER `, }}, }} @@ -471,11 +598,11 @@ func TestBuffering(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }, { @@ -490,8 +617,7 @@ func TestBuffering(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:ROW row_event: > > `, }, { `type:ROW row_event: > > `, @@ -499,6 +625,7 @@ func TestBuffering(t *testing.T) { `type:ROW row_event: > > `, }, { `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }, { @@ -511,13 +638,13 @@ func TestBuffering(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:ROW row_event: > > `, }, { `type:ROW row_event: > > `, }, { `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }, { @@ -529,11 +656,11 @@ func TestBuffering(t *testing.T) { "commit", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:ROW row_event: > > `, }, { `type:ROW row_event: after: > > `, + `gtid`, `commit`, }}, }, { @@ -580,19 +707,19 @@ func TestBestEffortNameInFieldEvent(t *testing.T) { // In this case, we don't have information about vitess_test since it was renamed to vitess_test_test. // information returned by binlog for val column == varchar (rather than varbinary). output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }, { - `gtid|begin`, + `gtid`, `type:DDL ddl:"rename table vitess_test to vitess_test_new" `, }, { - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }} @@ -626,8 +753,7 @@ func TestTypes(t *testing.T) { "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: ` + `fields: ` + @@ -653,6 +779,7 @@ func TestTypes(t *testing.T) { `18446744073709551615` + `2012` + `" > > > `, + `gtid`, `commit`, }}, }, { @@ -660,8 +787,7 @@ func TestTypes(t *testing.T) { "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: ` + `fields: ` + @@ -675,6 +801,7 @@ func TestTypes(t *testing.T) { `3.99E+00` + `4.99E+00` + `" > > > `, + `gtid`, `commit`, }}, }, { @@ -683,8 +810,7 @@ func TestTypes(t *testing.T) { "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: ` + `fields: ` + @@ -698,6 +824,7 @@ func TestTypes(t *testing.T) { `fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }, { @@ -706,8 +833,7 @@ func TestTypes(t *testing.T) { "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: ` + `fields: ` + @@ -723,6 +849,7 @@ func TestTypes(t *testing.T) { `15:45:45` + `\000\000\000\000\001\001\000\000\000\000\000\000\000\000\000\360?\000\000\000\000\000\000\000@` + `" > > > `, + `gtid`, `commit`, }}, }, { @@ -730,10 +857,10 @@ func TestTypes(t *testing.T) { "insert into vitess_null values(1, null)", }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }} @@ -759,10 +886,10 @@ func TestJSON(t *testing.T) { `insert into vitess_json values(1, '{"foo": "bar"}')`, }, output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, `type:FIELD field_event: fields: > `, `type:ROW row_event: > > `, + `gtid`, `commit`, }}, }} @@ -791,8 +918,47 @@ func TestExternalTable(t *testing.T) { }, // External table events don't get sent. output: [][]string{{ - `gtid|begin`, - `gtid|begin`, + `begin`, + `gtid`, + `commit`, + }}, + }} + runCases(t, nil, testcases, "") +} + +func TestJournal(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table _vt.resharding_journal(id int, db_name varchar(128), val blob, primary key(id))", + }) + defer execStatements(t, []string{ + "drop table _vt.resharding_journal", + }) + engine.se.Reload(context.Background()) + + journal1 := &binlogdatapb.Journal{ + Id: 1, + MigrationType: binlogdatapb.MigrationType_SHARDS, + } + journal2 := &binlogdatapb.Journal{ + Id: 2, + MigrationType: binlogdatapb.MigrationType_SHARDS, + } + testcases := []testcase{{ + input: []string{ + "begin", + fmt.Sprintf("insert into _vt.resharding_journal values(1, 'vttest', '%v')", journal1.String()), + fmt.Sprintf("insert into _vt.resharding_journal values(2, 'nosend', '%v')", journal2.String()), + "commit", + }, + // External table events don't get sent. + output: [][]string{{ + `begin`, + `type:JOURNAL journal: `, + `gtid`, `commit`, }}, }} @@ -842,39 +1008,38 @@ func TestStatementMode(t *testing.T) { if testing.Short() { t.Skip() } - execStatements(t, []string{ - "create table t1(id int, val1 varbinary(128), val2 varbinary(128), primary key(id))", - "insert into t1 values(1, 'aaa', 'bbb')", - }) - defer execStatements(t, []string{ - "drop table t1", + "create table stream1(id int, val varbinary(128), primary key(id))", + "create table stream2(id int, val varbinary(128), primary key(id))", }) + engine.se.Reload(context.Background()) - // Record position before the next few statements. - pos := masterPosition(t) - execStatements(t, []string{ - "set @@session.binlog_format='statement'", - "update t1 set val1='bbb' where id=1", - "set @@session.binlog_format='row'", + defer execStatements(t, []string{ + "drop table stream1", + "drop table stream2", }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ch := make(chan []*binlogdatapb.VEvent) - go func() { - for evs := range ch { - t.Errorf("received: %v", evs) - } - }() - defer close(ch) - err := vstream(ctx, t, pos, nil, ch) - want := "unexpected statement type" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("err: %v, must contain '%s'", err, want) - } + testcases := []testcase{{ + input: []string{ + "set @@session.binlog_format='STATEMENT'", + "begin", + "insert into stream1 values (1, 'aaa')", + "update stream1 set val='bbb' where id = 1", + "delete from stream1 where id = 1", + "commit", + "set @@session.binlog_format='ROW'", + }, + output: [][]string{{ + `begin`, + `type:INSERT dml:"insert into stream1 values (1, 'aaa')" `, + `type:UPDATE dml:"update stream1 set val='bbb' where id = 1" `, + `type:DELETE dml:"delete from stream1 where id = 1" `, + `gtid`, + `commit`, + }}, + }} + runCases(t, nil, testcases, "") } func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, postion string) { @@ -920,8 +1085,8 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ // CurrentTime is not testable. evs[i].CurrentTime = 0 switch want { - case "gtid|begin": - if evs[i].Type != binlogdatapb.VEventType_GTID && evs[i].Type != binlogdatapb.VEventType_BEGIN { + case "begin": + if evs[i].Type != binlogdatapb.VEventType_BEGIN { t.Fatalf("%v (%d): event: %v, want gtid or begin", input, i, evs[i]) } case "gtid": @@ -995,7 +1160,15 @@ func execStatements(t *testing.T, queries []string) { func masterPosition(t *testing.T) string { t.Helper() - pos, err := env.Mysqld.MasterPosition() + // We use the engine's cp because there is one test that overrides + // the flavor to FilePos. If so, we have to obtain the position + // in that flavor format. + conn, err := mysql.Connect(context.Background(), engine.cp) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + pos, err := conn.MasterPosition() if err != nil { t.Fatal(err) } diff --git a/go/vt/vttest/environment.go b/go/vt/vttest/environment.go index 336e92c3b06..130791bce4b 100644 --- a/go/vt/vttest/environment.go +++ b/go/vt/vttest/environment.go @@ -115,26 +115,11 @@ func GetMySQLOptions(flavor string) (string, []string, error) { flavor = DefaultMySQLFlavor } - mycnf := []string{"config/mycnf/vtcombo.cnf"} - switch flavor { - case "MariaDB103": - mycnf = append(mycnf, "config/mycnf/default-fast.cnf") - mycnf = append(mycnf, "config/mycnf/master_mariadb103.cnf") - case "MariaDB": - mycnf = append(mycnf, "config/mycnf/default-fast.cnf") - mycnf = append(mycnf, "config/mycnf/master_mariadb100.cnf") - case "MySQL80": - mycnf = append(mycnf, "config/mycnf/default-fast.cnf") - mycnf = append(mycnf, "config/mycnf/master_mysql80.cnf") - case "MySQL56": - mycnf = append(mycnf, "config/mycnf/default-fast.cnf") - mycnf = append(mycnf, "config/mycnf/master_mysql56.cnf") - default: - return "", nil, fmt.Errorf("unknown mysql flavor: %s", flavor) - } + mycnf := []string{} + mycnf = append(mycnf, "config/mycnf/default-fast.cnf") for i, cnf := range mycnf { - mycnf[i] = path.Join(os.Getenv("VTTOP"), cnf) + mycnf[i] = path.Join(os.Getenv("VTROOT"), cnf) } return flavor, mycnf, nil @@ -154,7 +139,7 @@ func (env *LocalTestEnv) BinaryPath(binary string) string { func (env *LocalTestEnv) MySQLManager(mycnf []string, snapshot string) (MySQLManager, error) { return &Mysqlctl{ Binary: env.BinaryPath("mysqlctl"), - InitFile: path.Join(os.Getenv("VTTOP"), "config/init_db.sql"), + InitFile: path.Join(os.Getenv("VTROOT"), "config/init_db.sql"), Directory: env.TmpPath, Port: env.PortForProtocol("mysql", ""), MyCnf: append(env.DefaultMyCnf, mycnf...), diff --git a/go/vt/vttls/vttls.go b/go/vt/vttls/vttls.go index 1b24192e797..65c8724d95b 100644 --- a/go/vt/vttls/vttls.go +++ b/go/vt/vttls/vttls.go @@ -19,8 +19,12 @@ package vttls import ( "crypto/tls" "crypto/x509" - "fmt" "io/ioutil" + "strings" + "sync" + + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" ) // Updated list of acceptable cipher suits to address @@ -51,6 +55,8 @@ func newTLSConfig() *tls.Config { } } +var onceByKeys = sync.Map{} + // ClientConfig returns the TLS config to use for a client to // connect to a server with the provided parameters. func ClientConfig(cert, key, ca, name string) (*tls.Config, error) { @@ -58,24 +64,24 @@ func ClientConfig(cert, key, ca, name string) (*tls.Config, error) { // Load the client-side cert & key if any. if cert != "" && key != "" { - crt, err := tls.LoadX509KeyPair(cert, key) + certificates, err := loadTLSCertificate(cert, key) + if err != nil { - return nil, fmt.Errorf("failed to load cert/key: %v", err) + return nil, err } - config.Certificates = []tls.Certificate{crt} + + config.Certificates = *certificates } // Load the server CA if any. if ca != "" { - b, err := ioutil.ReadFile(ca) + certificatePool, err := loadx509CertPool(ca) + if err != nil { - return nil, fmt.Errorf("failed to read ca file: %v", err) - } - cp := x509.NewCertPool() - if !cp.AppendCertsFromPEM(b) { - return nil, fmt.Errorf("failed to append certificates") + return nil, err } - config.RootCAs = cp + + config.RootCAs = certificatePool } // Set the server name if any. @@ -91,27 +97,109 @@ func ClientConfig(cert, key, ca, name string) (*tls.Config, error) { func ServerConfig(cert, key, ca string) (*tls.Config, error) { config := newTLSConfig() - // Load the server cert and key. - crt, err := tls.LoadX509KeyPair(cert, key) + certificates, err := loadTLSCertificate(cert, key) + if err != nil { - return nil, fmt.Errorf("failed to load cert/key: %v", err) + return nil, err } - config.Certificates = []tls.Certificate{crt} + + config.Certificates = *certificates // if specified, load ca to validate client, // and enforce clients present valid certs. if ca != "" { - b, err := ioutil.ReadFile(ca) + certificatePool, err := loadx509CertPool(ca) + if err != nil { - return nil, fmt.Errorf("failed to read ca file: %v", err) - } - cp := x509.NewCertPool() - if !cp.AppendCertsFromPEM(b) { - return nil, fmt.Errorf("failed to append certificates") + return nil, err } - config.ClientCAs = cp + + config.ClientCAs = certificatePool config.ClientAuth = tls.RequireAndVerifyClientCert } return config, nil } + +var certPools = sync.Map{} + +func loadx509CertPool(ca string) (*x509.CertPool, error) { + once, _ := onceByKeys.LoadOrStore(ca, &sync.Once{}) + + var err error + once.(*sync.Once).Do(func() { + err = doLoadx509CertPool(ca) + }) + if err != nil { + return nil, err + } + + result, ok := certPools.Load(ca) + + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "Cannot find loaded x509 cert pool for ca: %s", ca) + } + + return result.(*x509.CertPool), nil +} + +func doLoadx509CertPool(ca string) error { + b, err := ioutil.ReadFile(ca) + if err != nil { + return vterrors.Errorf(vtrpc.Code_NOT_FOUND, "failed to read ca file: %s", ca) + } + + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return vterrors.Errorf(vtrpc.Code_UNKNOWN, "failed to append certificates") + } + + certPools.Store(ca, cp) + + return nil +} + +var tlsCertificates = sync.Map{} + +func tlsCertificatesIdentifier(cert, key string) string { + return strings.Join([]string{cert, key}, ";") +} + +func loadTLSCertificate(cert, key string) (*[]tls.Certificate, error) { + tlsIdentifier := tlsCertificatesIdentifier(cert, key) + once, _ := onceByKeys.LoadOrStore(tlsIdentifier, &sync.Once{}) + + var err error + once.(*sync.Once).Do(func() { + err = doLoadTLSCertificate(cert, key) + }) + + if err != nil { + return nil, err + } + + result, ok := tlsCertificates.Load(tlsIdentifier) + + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "Cannot find loaded tls certificate with cert: %s, key%s", cert, key) + } + + return result.(*[]tls.Certificate), nil +} + +func doLoadTLSCertificate(cert, key string) error { + tlsIdentifier := tlsCertificatesIdentifier(cert, key) + + var certificate []tls.Certificate + // Load the server cert and key. + crt, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return vterrors.Errorf(vtrpc.Code_NOT_FOUND, "failed to load tls certificate, cert %s, key: %s", cert, key) + } + + certificate = []tls.Certificate{crt} + + tlsCertificates.Store(tlsIdentifier, &certificate) + + return nil +} diff --git a/go/vt/worker/key_resolver.go b/go/vt/worker/key_resolver.go index bc53eafd1c3..a79fb1b66f6 100644 --- a/go/vt/worker/key_resolver.go +++ b/go/vt/worker/key_resolver.go @@ -93,7 +93,7 @@ func (r *v2Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) { // table. type v3Resolver struct { shardingColumnIndex int - vindex vindexes.Vindex + vindex vindexes.SingleColumn } // newV3ResolverFromTableDefinition returns a keyspaceIDResolver for a v3 table. @@ -119,7 +119,8 @@ func newV3ResolverFromTableDefinition(keyspaceSchema *vindexes.KeyspaceSchema, t return &v3Resolver{ shardingColumnIndex: columnIndex, - vindex: colVindex.Vindex, + // Only SingleColumn vindexes are returned by FindVindexForSharding. + vindex: colVindex.Vindex.(vindexes.SingleColumn), }, nil } @@ -149,7 +150,8 @@ func newV3ResolverFromColumnList(keyspaceSchema *vindexes.KeyspaceSchema, name s return &v3Resolver{ shardingColumnIndex: columnIndex, - vindex: colVindex.Vindex, + // Only SingleColumn vindexes are returned by FindVindexForSharding. + vindex: colVindex.Vindex.(vindexes.SingleColumn), }, nil } diff --git a/go/vt/workflow/long_polling.go b/go/vt/workflow/long_polling.go index 85463a71a7d..007472a2b7b 100644 --- a/go/vt/workflow/long_polling.go +++ b/go/vt/workflow/long_polling.go @@ -249,7 +249,7 @@ func (m *Manager) HandleHTTPLongPolling(pattern string) { ctx := context.TODO() if err := m.NodeManager().Action(ctx, ap); err != nil { - return fmt.Errorf("Action failed: %v", err) + return fmt.Errorf("action failed: %v", err) } http.Error(w, "", http.StatusOK) return nil diff --git a/go/vt/workflow/node.go b/go/vt/workflow/node.go index 7043370aeef..7895a3ba321 100644 --- a/go/vt/workflow/node.go +++ b/go/vt/workflow/node.go @@ -441,7 +441,7 @@ func (m *NodeManager) Action(ctx context.Context, ap *ActionParameters) error { if n.Listener == nil { m.mu.Unlock() - return fmt.Errorf("Action %v is invoked on a node without listener (node path is %v)", ap.Name, ap.Path) + return fmt.Errorf("action %v is invoked on a node without listener (node path is %v)", ap.Name, ap.Path) } nodeListener := n.Listener m.mu.Unlock() diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 25e266df0dd..7a53014f464 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -90,6 +90,46 @@ func (wr *Wrangler) SetKeyspaceShardingInfo(ctx context.Context, keyspace, shard return wr.ts.UpdateKeyspace(ctx, ki) } +// validateNewWorkflow ensures that the specified workflow doesn't already exist +// in the keyspace. +func (wr *Wrangler) validateNewWorkflow(ctx context.Context, keyspace, workflow string) error { + allshards, err := wr.ts.FindAllShardsInKeyspace(ctx, keyspace) + if err != nil { + return err + } + var wg sync.WaitGroup + allErrors := &concurrency.AllErrorRecorder{} + for _, si := range allshards { + if si.MasterAlias == nil { + allErrors.RecordError(fmt.Errorf("shard has no master: %v", si)) + continue + } + wg.Add(1) + go func(si *topo.ShardInfo) { + defer wg.Done() + + master, err := wr.ts.GetTablet(ctx, si.MasterAlias) + if err != nil { + allErrors.RecordError(vterrors.Wrap(err, "validateWorkflowName.GetTablet")) + return + } + + query := fmt.Sprintf("select 1 from _vt.vreplication where db_name=%s and workflow=%s", encodeString(master.DbName()), encodeString(workflow)) + p3qr, err := wr.tmc.VReplicationExec(ctx, master.Tablet, query) + if err != nil { + allErrors.RecordError(vterrors.Wrap(err, "validateWorkflowName.VReplicationExec")) + return + } + if len(p3qr.Rows) != 0 { + allErrors.RecordError(fmt.Errorf("workflow %s already exists in keyspace %s", workflow, keyspace)) + return + } + }(si) + } + wg.Wait() + return allErrors.AggrError(vterrors.Aggregate) +} + // SplitClone initiates a SplitClone workflow. func (wr *Wrangler) SplitClone(ctx context.Context, keyspace string, from, to []string) error { var fromShards, toShards []*topo.ShardInfo diff --git a/go/vt/wrangler/migrater_env_test.go b/go/vt/wrangler/migrater_env_test.go index 5ead8aea8cb..81cdc7a6a1e 100644 --- a/go/vt/wrangler/migrater_env_test.go +++ b/go/vt/wrangler/migrater_env_test.go @@ -85,9 +85,6 @@ func newTestTableMigraterCustom(ctx context.Context, t *testing.T, sourceShards, if err != nil { t.Fatal(err) } - if sourceKeyRange == nil { - sourceKeyRange = &topodatapb.KeyRange{} - } tme.sourceKeyRanges = append(tme.sourceKeyRanges, sourceKeyRange) } for _, shard := range targetShards { @@ -98,9 +95,6 @@ func newTestTableMigraterCustom(ctx context.Context, t *testing.T, sourceShards, if err != nil { t.Fatal(err) } - if targetKeyRange == nil { - targetKeyRange = &topodatapb.KeyRange{} - } tme.targetKeyRanges = append(tme.targetKeyRanges, targetKeyRange) } @@ -209,9 +203,6 @@ func newTestShardMigrater(ctx context.Context, t *testing.T, sourceShards, targe if err != nil { t.Fatal(err) } - if sourceKeyRange == nil { - sourceKeyRange = &topodatapb.KeyRange{} - } tme.sourceKeyRanges = append(tme.sourceKeyRanges, sourceKeyRange) } for _, shard := range targetShards { @@ -222,9 +213,6 @@ func newTestShardMigrater(ctx context.Context, t *testing.T, sourceShards, targe if err != nil { t.Fatal(err) } - if targetKeyRange == nil { - targetKeyRange = &topodatapb.KeyRange{} - } tme.targetKeyRanges = append(tme.targetKeyRanges, targetKeyRange) } diff --git a/go/vt/wrangler/resharder.go b/go/vt/wrangler/resharder.go new file mode 100644 index 00000000000..d03a1d87eef --- /dev/null +++ b/go/vt/wrangler/resharder.go @@ -0,0 +1,362 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrangler + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/golang/protobuf/proto" + "github.com/pkg/errors" + "golang.org/x/net/context" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/binlog/binlogplayer" + "vitess.io/vitess/go/vt/concurrency" + "vitess.io/vitess/go/vt/key" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + "vitess.io/vitess/go/vt/throttler" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +type resharder struct { + wr *Wrangler + keyspace string + workflow string + sourceShards []*topo.ShardInfo + sourceMasters map[string]*topo.TabletInfo + targetShards []*topo.ShardInfo + targetMasters map[string]*topo.TabletInfo + vschema *vschemapb.Keyspace + refStreams map[string]*refStream +} + +type refStream struct { + workflow string + bls *binlogdatapb.BinlogSource + cell string + tabletTypes string +} + +// Reshard initiates a resharding workflow. +func (wr *Wrangler) Reshard(ctx context.Context, keyspace, workflow string, sources, targets []string, skipSchemaCopy bool) error { + if err := wr.validateNewWorkflow(ctx, keyspace, workflow); err != nil { + return err + } + + rs, err := wr.buildResharder(ctx, keyspace, workflow, sources, targets) + if err != nil { + return vterrors.Wrap(err, "buildResharder") + } + if !skipSchemaCopy { + if err := rs.copySchema(ctx); err != nil { + return vterrors.Wrap(err, "copySchema") + } + } + if err := rs.createStreams(ctx); err != nil { + return vterrors.Wrap(err, "createStreams") + } + if err := rs.startStreams(ctx); err != nil { + return vterrors.Wrap(err, "startStream") + } + return nil +} + +func (wr *Wrangler) buildResharder(ctx context.Context, keyspace, workflow string, sources, targets []string) (*resharder, error) { + rs := &resharder{ + wr: wr, + keyspace: keyspace, + workflow: workflow, + sourceMasters: make(map[string]*topo.TabletInfo), + targetMasters: make(map[string]*topo.TabletInfo), + } + for _, shard := range sources { + si, err := wr.ts.GetShard(ctx, keyspace, shard) + if err != nil { + return nil, vterrors.Wrapf(err, "GetShard(%s) failed", shard) + } + if !si.IsMasterServing { + return nil, fmt.Errorf("source shard %v is not in serving state", shard) + } + rs.sourceShards = append(rs.sourceShards, si) + master, err := wr.ts.GetTablet(ctx, si.MasterAlias) + if err != nil { + return nil, vterrors.Wrapf(err, "GetTablet(%s) failed", si.MasterAlias) + } + rs.sourceMasters[si.ShardName()] = master + } + for _, shard := range targets { + si, err := wr.ts.GetShard(ctx, keyspace, shard) + if err != nil { + return nil, vterrors.Wrapf(err, "GetShard(%s) failed", shard) + } + if si.IsMasterServing { + return nil, fmt.Errorf("target shard %v is in serving state", shard) + } + rs.targetShards = append(rs.targetShards, si) + master, err := wr.ts.GetTablet(ctx, si.MasterAlias) + if err != nil { + return nil, vterrors.Wrapf(err, "GetTablet(%s) failed", si.MasterAlias) + } + rs.targetMasters[si.ShardName()] = master + } + if err := topotools.ValidateForReshard(rs.sourceShards, rs.targetShards); err != nil { + return nil, vterrors.Wrap(err, "ValidateForReshard") + } + if err := rs.validateTargets(ctx); err != nil { + return nil, vterrors.Wrap(err, "validateTargets") + } + + vschema, err := wr.ts.GetVSchema(ctx, keyspace) + if err != nil { + return nil, vterrors.Wrap(err, "GetVSchema") + } + rs.vschema = vschema + + if err := rs.readRefStreams(ctx); err != nil { + return nil, vterrors.Wrap(err, "readRefStreams") + } + return rs, nil +} + +func (rs *resharder) validateTargets(ctx context.Context) error { + err := rs.forAll(rs.targetShards, func(target *topo.ShardInfo) error { + targetMaster := rs.targetMasters[target.ShardName()] + query := fmt.Sprintf("select 1 from _vt.vreplication where db_name=%s", encodeString(targetMaster.DbName())) + p3qr, err := rs.wr.tmc.VReplicationExec(ctx, targetMaster.Tablet, query) + if err != nil { + return vterrors.Wrapf(err, "VReplicationExec(%v, %s)", targetMaster.Tablet, query) + } + if len(p3qr.Rows) != 0 { + return errors.New("some streams already exist in the target shards, please clean them up and retry the command") + } + return nil + }) + return err +} + +func (rs *resharder) readRefStreams(ctx context.Context) error { + var mu sync.Mutex + err := rs.forAll(rs.sourceShards, func(source *topo.ShardInfo) error { + sourceMaster := rs.sourceMasters[source.ShardName()] + + query := fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name=%s", encodeString(sourceMaster.DbName())) + p3qr, err := rs.wr.tmc.VReplicationExec(ctx, sourceMaster.Tablet, query) + if err != nil { + return vterrors.Wrapf(err, "VReplicationExec(%v, %s)", sourceMaster.Tablet, query) + } + qr := sqltypes.Proto3ToResult(p3qr) + + mu.Lock() + defer mu.Unlock() + + mustCreate := false + var ref map[string]bool + if rs.refStreams == nil { + rs.refStreams = make(map[string]*refStream) + mustCreate = true + } else { + // Copy the ref streams for comparison. + ref = make(map[string]bool, len(rs.refStreams)) + for k := range rs.refStreams { + ref[k] = true + } + } + for _, row := range qr.Rows { + workflow := row[0].ToString() + if workflow == "" { + return fmt.Errorf("VReplication streams must have named workflows for migration: shard: %s:%s", source.Keyspace(), source.ShardName()) + } + var bls binlogdatapb.BinlogSource + if err := proto.UnmarshalText(row[1].ToString(), &bls); err != nil { + return vterrors.Wrapf(err, "UnmarshalText: %v", row) + } + isReference, err := rs.blsIsReference(&bls) + if err != nil { + return vterrors.Wrap(err, "blsIsReference") + } + if !isReference { + continue + } + key := fmt.Sprintf("%s:%s:%s", workflow, bls.Keyspace, bls.Shard) + if mustCreate { + rs.refStreams[key] = &refStream{ + workflow: workflow, + bls: &bls, + cell: row[2].ToString(), + tabletTypes: row[3].ToString(), + } + } else { + if !ref[key] { + return fmt.Errorf("streams are mismatched across source shards for workflow: %s", workflow) + } + delete(ref, key) + } + } + if len(ref) != 0 { + return fmt.Errorf("streams are mismatched across source shards: %v", ref) + } + return nil + }) + return err +} + +// blsIsReference is partially copied from streamMigrater.templatize. +// It reuses the constants from that function also. +func (rs *resharder) blsIsReference(bls *binlogdatapb.BinlogSource) (bool, error) { + streamType := unknown + for _, rule := range bls.Filter.Rules { + typ, err := rs.identifyRuleType(rule) + if err != nil { + return false, err + } + switch typ { + case sharded: + if streamType == reference { + return false, fmt.Errorf("cannot reshard streams with a mix of reference and sharded tables: %v", bls) + } + streamType = sharded + case reference: + if streamType == sharded { + return false, fmt.Errorf("cannot reshard streams with a mix of reference and sharded tables: %v", bls) + } + streamType = reference + } + } + return streamType == reference, nil +} + +func (rs *resharder) identifyRuleType(rule *binlogdatapb.Rule) (int, error) { + vtable, ok := rs.vschema.Tables[rule.Match] + if !ok { + return 0, fmt.Errorf("table %v not found in vschema", rule.Match) + } + if vtable.Type == vindexes.TypeReference { + return reference, nil + } + // In this case, 'sharded' means that it's not a reference + // table. We don't care about any other subtleties. + return sharded, nil +} + +func (rs *resharder) copySchema(ctx context.Context) error { + oneSource := rs.sourceShards[0].MasterAlias + err := rs.forAll(rs.targetShards, func(target *topo.ShardInfo) error { + return rs.wr.CopySchemaShard(ctx, oneSource, []string{"/.*"}, nil, false, rs.keyspace, target.ShardName(), 1*time.Second) + }) + return err +} + +func (rs *resharder) createStreams(ctx context.Context) error { + var excludeRules []*binlogdatapb.Rule + for tableName, table := range rs.vschema.Tables { + if table.Type == vindexes.TypeReference { + excludeRules = append(excludeRules, &binlogdatapb.Rule{ + Match: tableName, + Filter: "exclude", + }) + } + } + + err := rs.forAll(rs.targetShards, func(target *topo.ShardInfo) error { + targetMaster := rs.targetMasters[target.ShardName()] + + buf := &strings.Builder{} + buf.WriteString("insert into _vt.vreplication(workflow, source, pos, max_tps, max_replication_lag, cell, tablet_types, time_updated, transaction_timestamp, state, db_name) values ") + prefix := "" + + addLine := func(workflow string, bls *binlogdatapb.BinlogSource, cell, tabletTypes string) { + fmt.Fprintf(buf, "%s(%v, %v, '', %v, %v, %v, %v, %v, 0, '%v', %v)", + prefix, + encodeString(workflow), + encodeString(bls.String()), + throttler.MaxRateModuleDisabled, + throttler.ReplicationLagModuleDisabled, + encodeString(cell), + encodeString(tabletTypes), + time.Now().Unix(), + binlogplayer.BlpStopped, + encodeString(targetMaster.DbName())) + prefix = ", " + } + + // copy excludeRules to prevent data race. + copyExcludeRules := append([]*binlogdatapb.Rule(nil), excludeRules...) + for _, source := range rs.sourceShards { + if !key.KeyRangesIntersect(target.KeyRange, source.KeyRange) { + continue + } + filter := &binlogdatapb.Filter{ + Rules: append(copyExcludeRules, &binlogdatapb.Rule{ + Match: "/.*", + Filter: key.KeyRangeString(target.KeyRange), + }), + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: rs.keyspace, + Shard: source.ShardName(), + Filter: filter, + } + addLine(rs.workflow, bls, "", "") + } + + for _, rstream := range rs.refStreams { + addLine(rstream.workflow, rstream.bls, rstream.cell, rstream.tabletTypes) + } + query := buf.String() + if _, err := rs.wr.tmc.VReplicationExec(ctx, targetMaster.Tablet, query); err != nil { + return vterrors.Wrapf(err, "VReplicationExec(%v, %s)", targetMaster.Tablet, query) + } + return nil + }) + + return err +} + +func (rs *resharder) startStreams(ctx context.Context) error { + err := rs.forAll(rs.targetShards, func(target *topo.ShardInfo) error { + targetMaster := rs.targetMasters[target.ShardName()] + query := fmt.Sprintf("update _vt.vreplication set state='Running' where db_name=%s", encodeString(targetMaster.DbName())) + if _, err := rs.wr.tmc.VReplicationExec(ctx, targetMaster.Tablet, query); err != nil { + return vterrors.Wrapf(err, "VReplicationExec(%v, %s)", targetMaster.Tablet, query) + } + return nil + }) + return err +} + +func (rs *resharder) forAll(shards []*topo.ShardInfo, f func(*topo.ShardInfo) error) error { + var wg sync.WaitGroup + allErrors := &concurrency.AllErrorRecorder{} + for _, shard := range shards { + wg.Add(1) + go func(shard *topo.ShardInfo) { + defer wg.Done() + + if err := f(shard); err != nil { + allErrors.RecordError(err) + } + }(shard) + } + wg.Wait() + return allErrors.AggrError(vterrors.Aggregate) +} diff --git a/go/vt/wrangler/resharder_env_test.go b/go/vt/wrangler/resharder_env_test.go new file mode 100644 index 00000000000..dfdb156cc33 --- /dev/null +++ b/go/vt/wrangler/resharder_env_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrangler + +import ( + "fmt" + "regexp" + "sync" + "testing" + + "golang.org/x/net/context" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/logutil" + querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vttablet/tmclient" +) + +type testResharderEnv struct { + wr *Wrangler + keyspace string + workflow string + sources []string + targets []string + tablets map[int]*topodatapb.Tablet + topoServ *topo.Server + cell string + tmc *testResharderTMClient +} + +//---------------------------------------------- +// testResharderEnv + +func newTestResharderEnv(sources, targets []string) *testResharderEnv { + env := &testResharderEnv{ + keyspace: "ks", + workflow: "resharderTest", + sources: sources, + targets: targets, + tablets: make(map[int]*topodatapb.Tablet), + topoServ: memorytopo.NewServer("cell"), + cell: "cell", + tmc: newTestResharderTMClient(), + } + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + + tabletID := 100 + for _, shard := range sources { + _ = env.addTablet(tabletID, env.keyspace, shard, topodatapb.TabletType_MASTER) + tabletID += 10 + } + tabletID = 200 + for _, shard := range targets { + _ = env.addTablet(tabletID, env.keyspace, shard, topodatapb.TabletType_MASTER) + tabletID += 10 + } + return env +} + +func (env *testResharderEnv) expectValidation() { + for _, tablet := range env.tablets { + tabletID := int(tablet.Alias.Uid) + // wr.validateNewWorkflow + env.tmc.expectVRQuery(tabletID, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + + if tabletID >= 200 { + // validateTargets + env.tmc.expectVRQuery(tabletID, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s'", env.keyspace), &sqltypes.Result{}) + } + } +} + +func (env *testResharderEnv) expectNoRefStream() { + for _, tablet := range env.tablets { + tabletID := int(tablet.Alias.Uid) + if tabletID < 200 { + // readRefStreams + env.tmc.expectVRQuery(tabletID, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), &sqltypes.Result{}) + } + } +} + +func (env *testResharderEnv) close() { + for _, t := range env.tablets { + env.deleteTablet(t) + } +} + +func (env *testResharderEnv) addTablet(id int, keyspace, shard string, tabletType topodatapb.TabletType) *topodatapb.Tablet { + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: env.cell, + Uid: uint32(id), + }, + Keyspace: keyspace, + Shard: shard, + KeyRange: &topodatapb.KeyRange{}, + Type: tabletType, + PortMap: map[string]int32{ + "test": int32(id), + }, + } + env.tablets[id] = tablet + if err := env.wr.InitTablet(context.Background(), tablet, false /* allowMasterOverride */, true /* createShardAndKeyspace */, false /* allowUpdate */); err != nil { + panic(err) + } + if tabletType == topodatapb.TabletType_MASTER { + _, err := env.wr.ts.UpdateShardFields(context.Background(), keyspace, shard, func(si *topo.ShardInfo) error { + si.MasterAlias = tablet.Alias + return nil + }) + if err != nil { + panic(err) + } + } + return tablet +} + +func (env *testResharderEnv) deleteTablet(tablet *topodatapb.Tablet) { + env.topoServ.DeleteTablet(context.Background(), tablet.Alias) + delete(env.tablets, int(tablet.Alias.Uid)) +} + +//---------------------------------------------- +// testResharderTMClient + +type testResharderTMClient struct { + tmclient.TabletManagerClient + schema *tabletmanagerdatapb.SchemaDefinition + + mu sync.Mutex + vrQueries map[int][]*queryResult +} + +type queryResult struct { + query string + result *querypb.QueryResult +} + +func newTestResharderTMClient() *testResharderTMClient { + return &testResharderTMClient{ + vrQueries: make(map[int][]*queryResult), + } +} + +func (tmc *testResharderTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { + return tmc.schema, nil +} + +func (tmc *testResharderTMClient) expectVRQuery(tabletID int, query string, result *sqltypes.Result) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + tmc.vrQueries[tabletID] = append(tmc.vrQueries[tabletID], &queryResult{ + query: query, + result: sqltypes.ResultToProto3(result), + }) +} + +func (tmc *testResharderTMClient) VReplicationExec(ctx context.Context, tablet *topodatapb.Tablet, query string) (*querypb.QueryResult, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + qrs := tmc.vrQueries[int(tablet.Alias.Uid)] + if len(qrs) == 0 { + return nil, fmt.Errorf("tablet %v does not expect any more queries: %s", tablet, query) + } + matched := false + if qrs[0].query[0] == '/' { + matched = regexp.MustCompile(qrs[0].query[1:]).MatchString(query) + } else { + matched = query == qrs[0].query + } + if !matched { + return nil, fmt.Errorf("tablet %v: unexpected query %s, want: %s", tablet, query, qrs[0].query) + } + tmc.vrQueries[int(tablet.Alias.Uid)] = qrs[1:] + return qrs[0].result, nil +} + +func (tmc *testResharderTMClient) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, query []byte, maxRows int, disableBinlogs, reloadSchema bool) (*querypb.QueryResult, error) { + // Reuse VReplicationExec + return tmc.VReplicationExec(ctx, tablet, string(query)) +} + +func (tmc *testResharderTMClient) verifyQueries(t *testing.T) { + t.Helper() + + tmc.mu.Lock() + defer tmc.mu.Unlock() + + for tabletID, qrs := range tmc.vrQueries { + if len(qrs) != 0 { + var list []string + for _, qr := range qrs { + list = append(list, qr.query) + } + t.Errorf("tablet %v: has unreturned results: %v", tabletID, list) + } + } +} diff --git a/go/vt/wrangler/resharder_test.go b/go/vt/wrangler/resharder_test.go new file mode 100644 index 00000000000..d9214c8248d --- /dev/null +++ b/go/vt/wrangler/resharder_test.go @@ -0,0 +1,768 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrangler + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" + "vitess.io/vitess/go/sqltypes" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +const insertPrefix = `/insert into _vt.vreplication\(workflow, source, pos, max_tps, max_replication_lag, cell, tablet_types, time_updated, transaction_timestamp, state, db_name\) values ` +const eol = "$" + +func TestResharderOneToMany(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.expectValidation() + env.expectNoRefStream() + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +func TestResharderManyToOne(t *testing.T) { + env := newTestResharderEnv([]string{"-80", "80-"}, []string{"0"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.expectValidation() + env.expectNoRefStream() + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"-80\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\).*`+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"80-\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +func TestResharderManyToMany(t *testing.T) { + env := newTestResharderEnv([]string{"-40", "40-"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.expectValidation() + env.expectNoRefStream() + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"-40\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\).*`+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"40-\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"40-\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +// TestResharderOneRefTable tests the case where there's one ref table, but no stream for it. +// This means that the table is being updated manually. +func TestResharderOneRefTable(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + env.expectNoRefStream() + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: rules: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: rules: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +// TestResharderOneRefStream tests the case where there's one ref table and an associated stream. +func TestResharderOneRefStream(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + refRow := `\('t1', 'keyspace:\\"ks1\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, 'cell1', 'master,replica', [0-9]*, 0, 'Stopped', 'vt_ks'\)` + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: rules: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\).*`+ + refRow+eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: rules: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\).*`+ + refRow+eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +// TestResharderNoRefStream tests the case where there's a stream, but it's not a reference. +func TestResharderNoRefStream(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "hash": { + Type: "hash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "t1": { + ColumnVindexes: []*vschemapb.ColumnVindex{{ + Column: "c1", + Name: "hash", + }}, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t2", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +func TestResharderCopySchema(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.expectValidation() + env.expectNoRefStream() + + // These queries confirm that the copy schema function is getting called. + env.tmc.expectVRQuery(100, "SELECT 1 FROM information_schema.tables WHERE table_schema = '_vt' AND table_name = 'shard_metadata'", &sqltypes.Result{}) + env.tmc.expectVRQuery(100, "SELECT 1 FROM information_schema.tables WHERE table_schema = '_vt' AND table_name = 'shard_metadata'", &sqltypes.Result{}) + + env.tmc.expectVRQuery( + 200, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + env.tmc.expectVRQuery( + 210, + insertPrefix+ + `\('resharderTest', 'keyspace:\\"ks\\" shard:\\"0\\" filter: > ', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_ks'\)`+ + eol, + &sqltypes.Result{}, + ) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + env.tmc.expectVRQuery(210, "update _vt.vreplication set state='Running' where db_name='vt_ks'", &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, false) + assert.NoError(t, err) + env.tmc.verifyQueries(t) +} + +func TestResharderDupWorkflow(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.tmc.expectVRQuery(100, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "1", + "int64"), + "1", + ) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), result) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.EqualError(t, err, "workflow resharderTest already exists in keyspace ks") + env.tmc.verifyQueries(t) +} + +func TestResharderServingState(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.tmc.expectVRQuery(100, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, []string{"-80"}, nil, true) + assert.EqualError(t, err, "buildResharder: source shard -80 is not in serving state") + + env.tmc.expectVRQuery(100, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + err = env.wr.Reshard(context.Background(), env.keyspace, env.workflow, []string{"0"}, []string{"0"}, true) + assert.EqualError(t, err, "buildResharder: target shard 0 is in serving state") + + env.tmc.expectVRQuery(100, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + err = env.wr.Reshard(context.Background(), env.keyspace, env.workflow, []string{"0"}, []string{"-80"}, true) + assert.EqualError(t, err, "buildResharder: ValidateForReshard: source and target keyranges don't match: - vs -80") +} + +func TestResharderTargetAlreadyResharding(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.tmc.expectVRQuery(100, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s' and workflow='%s'", env.keyspace, env.workflow), &sqltypes.Result{}) + + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "1", + "int64"), + "1", + ) + env.tmc.expectVRQuery(200, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + env.tmc.expectVRQuery(210, fmt.Sprintf("select 1 from _vt.vreplication where db_name='vt_%s'", env.keyspace), &sqltypes.Result{}) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.EqualError(t, err, "buildResharder: validateTargets: some streams already exist in the target shards, please clean them up and retry the command") + env.tmc.verifyQueries(t) +} + +func TestResharderUnnamedStream(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.EqualError(t, err, "buildResharder: readRefStreams: VReplication streams must have named workflows for migration: shard: ks:0") + env.tmc.verifyQueries(t) +} + +func TestResharderMismatchedRefStreams(t *testing.T) { + env := newTestResharderEnv([]string{"-80", "80-"}, []string{"0"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls1 := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + }}, + }, + } + result1 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1|%v|cell1|master,replica", bls1), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result1) + bls2 := &binlogdatapb.BinlogSource{ + Keyspace: "ks2", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + }}, + }, + } + result2 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1|%v|cell1|master,replica", bls1), + fmt.Sprintf("t1|%v|cell1|master,replica", bls2), + ) + env.tmc.expectVRQuery(110, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result2) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + want := "buildResharder: readRefStreams: streams are mismatched across source shards" + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("Reshard err: %v, want %v", err, want) + } + env.tmc.verifyQueries(t) +} + +func TestResharderTableNotInVSchema(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + assert.EqualError(t, err, "buildResharder: readRefStreams: blsIsReference: table t1 not found in vschema") + env.tmc.verifyQueries(t) +} + +func TestResharderMixedTablesOrder1(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "hash": { + Type: "hash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "t1": { + ColumnVindexes: []*vschemapb.ColumnVindex{{ + Column: "c1", + Name: "hash", + }}, + }, + "t2": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t2", + }, { + Match: "t2", + Filter: "select * from t2", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1t2|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + want := "buildResharder: readRefStreams: blsIsReference: cannot reshard streams with a mix of reference and sharded tables" + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("Reshard err: %v, want %v", err.Error(), want) + } + env.tmc.verifyQueries(t) +} + +func TestResharderMixedTablesOrder2(t *testing.T) { + env := newTestResharderEnv([]string{"0"}, []string{"-80", "80-"}) + defer env.close() + + schm := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ + Name: "t1", + Columns: []string{"c1", "c2"}, + PrimaryKeyColumns: []string{"c1"}, + Fields: sqltypes.MakeTestFields("c1|c2", "int64|int64"), + }}, + } + env.tmc.schema = schm + + vs := &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "hash": { + Type: "hash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "t1": { + ColumnVindexes: []*vschemapb.ColumnVindex{{ + Column: "c1", + Name: "hash", + }}, + }, + "t2": { + Type: vindexes.TypeReference, + }, + }, + } + if err := env.wr.ts.SaveVSchema(context.Background(), env.keyspace, vs); err != nil { + t.Fatal(err) + } + + env.expectValidation() + + bls := &binlogdatapb.BinlogSource{ + Keyspace: "ks1", + Shard: "0", + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t2", + Filter: "select * from t2", + }, { + Match: "t1", + Filter: "select * from t2", + }}, + }, + } + result := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "workflow|source|cell|tablet_types", + "varchar|varchar|varchar|varchar"), + fmt.Sprintf("t1t2|%v|cell1|master,replica", bls), + ) + env.tmc.expectVRQuery(100, fmt.Sprintf("select workflow, source, cell, tablet_types from _vt.vreplication where db_name='vt_%s'", env.keyspace), result) + + err := env.wr.Reshard(context.Background(), env.keyspace, env.workflow, env.sources, env.targets, true) + want := "buildResharder: readRefStreams: blsIsReference: cannot reshard streams with a mix of reference and sharded tables" + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("Reshard err: %v, want %v", err.Error(), want) + } + env.tmc.verifyQueries(t) +} diff --git a/go/vt/wrangler/vdiff_env_test.go b/go/vt/wrangler/vdiff_env_test.go index f41db6315e8..44f4f9b85ee 100644 --- a/go/vt/wrangler/vdiff_env_test.go +++ b/go/vt/wrangler/vdiff_env_test.go @@ -19,6 +19,7 @@ package wrangler import ( "flag" "fmt" + "sync" "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" @@ -51,11 +52,13 @@ const ( type testVDiffEnv struct { wr *Wrangler workflow string - tablets map[int]*testVDiffTablet topoServ *topo.Server cell string tabletType topodatapb.TabletType tmc *testVDiffTMClient + + mu sync.Mutex + tablets map[int]*testVDiffTablet } // vdiffEnv has to be a global for RegisterDialer to work. @@ -63,6 +66,8 @@ var vdiffEnv *testVDiffEnv func init() { tabletconn.RegisterDialer("VDiffTest", func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { + vdiffEnv.mu.Lock() + defer vdiffEnv.mu.Unlock() return vdiffEnv.tablets[int(tablet.Alias.Uid)], nil }) } @@ -163,12 +168,17 @@ func newTestVDiffEnv(sourceShards, targetShards []string, query string, position } func (env *testVDiffEnv) close() { + env.mu.Lock() + defer env.mu.Unlock() for _, t := range env.tablets { - env.deleteTablet(t.tablet) + env.topoServ.DeleteTablet(context.Background(), t.tablet.Alias) } + env.tablets = nil } func (env *testVDiffEnv) addTablet(id int, keyspace, shard string, tabletType topodatapb.TabletType) *testVDiffTablet { + env.mu.Lock() + defer env.mu.Unlock() tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: env.cell, @@ -198,11 +208,6 @@ func (env *testVDiffEnv) addTablet(id int, keyspace, shard string, tabletType to return env.tablets[id] } -func (env *testVDiffEnv) deleteTablet(tablet *topodatapb.Tablet) { - env.topoServ.DeleteTablet(context.Background(), tablet.Alias) - delete(env.tablets, int(tablet.Alias.Uid)) -} - //---------------------------------------------- // testVDiffTablet diff --git a/helm/release.sh b/helm/release.sh index 7c8bb639b6a..191976f4c38 100755 --- a/helm/release.sh +++ b/helm/release.sh @@ -1,6 +1,6 @@ #!/bin/bash -version_tag=1.0.6 +version_tag=1.0.7-5 docker pull vitess/k8s:latest docker tag vitess/k8s:latest vitess/k8s:helm-$version_tag diff --git a/helm/vitess/CHANGELOG.md b/helm/vitess/CHANGELOG.md index 77c035acdfe..ca034d69536 100644 --- a/helm/vitess/CHANGELOG.md +++ b/helm/vitess/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.0.7-5 - 2019-12-02 + +### Changes +* Update images of Vitess components to v4.0.0 +* Update MySQL image to Percona 5.7.26 +* Support for OpenTracing + ## 1.0.6 - 2019-01-20 ### Changes diff --git a/helm/vitess/Chart.yaml b/helm/vitess/Chart.yaml index 16da802ec96..3ef992e038c 100644 --- a/helm/vitess/Chart.yaml +++ b/helm/vitess/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: vitess -version: 1.0.6 +version: 1.0.7-5 description: Single-Chart Vitess Cluster keywords: - vitess diff --git a/helm/vitess/README.md b/helm/vitess/README.md index 8e884b60117..bac75a0be22 100644 --- a/helm/vitess/README.md +++ b/helm/vitess/README.md @@ -392,7 +392,7 @@ metadata: data: extra.cnf: |- early-plugin-load=keyring_vault=keyring_vault.so - # this includes default rpl plugins, see https://github.com/vitessio/vitess/blob/master/config/mycnf/master_mysql56.cnf for details + # this includes default rpl plugins, see https://github.com/vitessio/vitess/blob/master/config/mycnf/master_mysql57.cnf for details plugin-load=rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so;keyring_udf=keyring_udf.so keyring_vault_config=/vt/usersecrets/vttablet-vault/vault.conf # load keyring configuration from secret innodb_encrypt_tables=ON # encrypt all tables by default @@ -419,3 +419,15 @@ vttablet: secrets: - vttablet-vault ``` + +### Enable tracing (opentracing-jaeger) + +To enable tracing using opentracing Jaeger of Vitess components add tracing config with tracer `opentracing-jaeger` to `extraFlags`. For example to enable tracing for `vtgate`: + +```yaml +vtgate: + extraFlags: + jaeger-agent-host: "JAEGER-AGENT:6831" + tracing-sampling-rate: 0.1 + tracer: opentracing-jaeger +``` \ No newline at end of file diff --git a/helm/vitess/templates/_orchestrator.tpl b/helm/vitess/templates/_orchestrator.tpl index 45b3f65f1a8..13ad07831aa 100644 --- a/helm/vitess/templates/_orchestrator.tpl +++ b/helm/vitess/templates/_orchestrator.tpl @@ -123,7 +123,7 @@ spec: value: "15999" - name: recovery-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH @@ -133,7 +133,7 @@ spec: mountPath: /tmp - name: audit-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH diff --git a/helm/vitess/templates/_pmm.tpl b/helm/vitess/templates/_pmm.tpl index b6b4bdf22ef..9669f58298d 100644 --- a/helm/vitess/templates/_pmm.tpl +++ b/helm/vitess/templates/_pmm.tpl @@ -219,7 +219,7 @@ spec: trap : TERM INT; sleep infinity & wait - name: pmm-client-metrics-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index ded11358c48..3053407af4d 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -534,7 +534,7 @@ spec: {{ define "cont-logrotate" }} - name: logrotate - image: vitess/logrotate:helm-1.0.6 + image: vitess/logrotate:helm-1.0.7-5 imagePullPolicy: IfNotPresent volumeMounts: - name: vtdataroot @@ -548,7 +548,7 @@ spec: {{ define "cont-mysql-errorlog" }} - name: error-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: @@ -566,7 +566,7 @@ spec: {{ define "cont-mysql-slowlog" }} - name: slow-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: @@ -584,7 +584,7 @@ spec: {{ define "cont-mysql-generallog" }} - name: general-log - image: vitess/logtail:helm-1.0.6 + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index 7bd53706d03..96ef944051c 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -33,43 +33,43 @@ config: # choose a backup service - valid values are gcs/s3 # TODO: add file and ceph support - # backup_storage_implementation: gcs + backup_storage_implementation: gcs ######### # gcs settings ######### # Google Cloud Storage bucket to use for backups - # gcs_backup_storage_bucket: vitess-backups + gcs_backup_storage_bucket: vitess-backups # root prefix for all backup-related object names - # gcs_backup_storage_root: vtbackups + gcs_backup_storage_root: vtbackups # secret that contains Google service account json with read/write access to the bucket # kubectl create secret generic vitess-backups-creds --from-file=gcp-creds.json # can be omitted if running on a GCE/GKE node with default permissions - # gcsSecret: vitess-gcs-creds + gcsSecret: vitess-gcs-creds ######### # s3 settings ######### # AWS region to use - # s3_backup_aws_region: us-east-1 + s3_backup_aws_region: us-east-1 # S3 bucket to use for backups - # s3_backup_storage_bucket: vitess-backups + s3_backup_storage_bucket: vitess-backups # root prefix for all backup-related object names - # s3_backup_storage_root: vtbackups + s3_backup_storage_root: vtbackups # server-side encryption algorithm (e.g., AES256, aws:kms) - # s3_backup_server_side_encryption: AES256 + s3_backup_server_side_encryption: AES256 # secret that contains AWS S3 credentials file with read/write access to the bucket # kubectl create secret generic s3-credentials --from-file=s3-creds # can be omitted if running on a node with default permissions - # s3Secret: vitess-s3-creds + s3Secret: vitess-s3-creds topology: globalCell: @@ -180,7 +180,7 @@ etcd: # Default values for vtctld resources defined in 'topology' vtctld: serviceType: ClusterIP - vitessTag: helm-1.0.6 + vitessTag: helm-1.0.7-5 resources: # requests: # cpu: 100m @@ -191,7 +191,7 @@ vtctld: # Default values for vtgate resources defined in 'topology' vtgate: serviceType: ClusterIP - vitessTag: helm-1.0.6 + vitessTag: helm-1.0.7-5 resources: # requests: # cpu: 500m @@ -201,22 +201,22 @@ vtgate: # The options below are the most commonly adjusted, but any flag can be put here. # run vtgate --help to see all available flags extraFlags: - # MySQL server version to advertise. (default "5.5.10-Vitess") - # If running 8.0, you may need to use something like "8.0.13-Vitess" + # MySQL server version to advertise. (default "5.7.9-Vitess") + # If running 8.0, you may prefer to use something like "8.0.13-Vitess" # to prevent db clients from running deprecated queries on startup - mysql_server_version: "5.5.10-Vitess" + mysql_server_version: "5.7.9-Vitess" secrets: [] # secrets are mounted under /vt/usersecrets/{secretname} # Default values for vtctlclient resources defined in 'topology' vtctlclient: - vitessTag: helm-1.0.6 + vitessTag: helm-1.0.7-5 extraFlags: {} secrets: [] # secrets are mounted under /vt/usersecrets/{secretname} # Default values for vtworker resources defined in 'jobs' vtworker: - vitessTag: helm-1.0.6 + vitessTag: helm-1.0.7-5 extraFlags: {} resources: # requests: @@ -227,7 +227,7 @@ vtworker: # Default values for vttablet resources defined in 'topology' vttablet: - vitessTag: helm-1.0.6 + vitessTag: helm-1.0.7-5 # valid values are # - mysql56 (for MySQL 8.0) @@ -237,7 +237,7 @@ vttablet: # the flavor determines the base my.cnf file for vitess to function flavor: mysql56 - mysqlImage: percona:5.7.23 + mysqlImage: percona:5.7.26 # mysqlImage: mysql:5.7.24 # mysqlImage: mariadb:10.3.11 diff --git a/java/client/pom.xml b/java/client/pom.xml index 48e4f42b747..571b8866cce 100644 --- a/java/client/pom.xml +++ b/java/client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT vitess-client diff --git a/java/client/src/test/java/io/vitess/client/TestEnv.java b/java/client/src/test/java/io/vitess/client/TestEnv.java index 1e631e8bfb4..1b12162fcce 100644 --- a/java/client/src/test/java/io/vitess/client/TestEnv.java +++ b/java/client/src/test/java/io/vitess/client/TestEnv.java @@ -72,13 +72,13 @@ public void setPythonScriptProcess(Process process) { * Get setup command to launch a cluster. */ public List getSetupCommand(int port) { - String vtTop = System.getenv("VTTOP"); - if (vtTop == null) { - throw new RuntimeException("cannot find env variable: VTTOP"); + String vtRoot = System.getenv("VTROOT"); + if (vtRoot == null) { + throw new RuntimeException("cannot find env variable: VTROOT"); } String schemaDir = getTestDataPath() + "/schema"; List command = new ArrayList(); - command.add(vtTop + "/py/vttest/run_local_database.py"); + command.add(vtRoot + "/py/vttest/run_local_database.py"); command.add("--port"); command.add(Integer.toString(port)); command.add("--proto_topo"); @@ -89,11 +89,11 @@ public List getSetupCommand(int port) { } public String getTestDataPath() { - String vtTop = System.getenv("VTTOP"); - if (vtTop == null) { - throw new RuntimeException("cannot find env variable: VTTOP"); + String vtRoot = System.getenv("VTROOT"); + if (vtRoot == null) { + throw new RuntimeException("cannot find env variable: VTROOT"); } - return vtTop + "/data/test"; + return vtRoot + "/data/test"; } public String getTestOutputPath() { diff --git a/java/example/pom.xml b/java/example/pom.xml index 86a5bf887de..47301f29bb0 100644 --- a/java/example/pom.xml +++ b/java/example/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT vitess-example diff --git a/java/grpc-client/pom.xml b/java/grpc-client/pom.xml index 0ec7f824104..1bf7ba2ee7f 100644 --- a/java/grpc-client/pom.xml +++ b/java/grpc-client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT vitess-grpc-client diff --git a/java/hadoop/pom.xml b/java/hadoop/pom.xml index c26e9cb0e3a..37e815dde24 100644 --- a/java/hadoop/pom.xml +++ b/java/hadoop/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT vitess-hadoop diff --git a/java/hadoop/src/main/java/io/vitess/hadoop/README.md b/java/hadoop/src/main/java/io/vitess/hadoop/README.md index fe88bdc650e..9e4a083bd4c 100644 --- a/java/hadoop/src/main/java/io/vitess/hadoop/README.md +++ b/java/hadoop/src/main/java/io/vitess/hadoop/README.md @@ -14,7 +14,7 @@ primary key (id)) Engine=InnoDB; Let's say we want to write a MapReduce job that imports this table from Vitess to HDFS where each row is turned into a CSV record in HDFS. -We can use [VitessInputFormat](https://github.com/vitessio/vitess/blob/master/java/hadoop/src/main/java/io/vitess/hadoop/VitessInputFormat.java), an implementation of Hadoop's [InputFormat](https://hadoop.apache.org/docs/stable/api/org/apache/hadoop/mapred/InputFormat.html), for that. With VitessInputFormat, rows from the source table are streamed to the mapper task. Each input record has a [NullWritable](https://hadoop.apache.org/docs/r2.2.0/api/org/apache/hadoop/io/NullWritable.html) key (no key, really), and [RowWritable](https://github.com/vitessio/vitess/blob/master/java/hadoop/src/main/java/io/vitess/hadoop/RowWritable.java) as value, which is a writable implementation for the entire row's contents. +We can use [VitessInputFormat](https://github.com/vitessio/vitess/blob/master/java/hadoop/src/main/java/io/vitess/hadoop/VitessInputFormat.java), an implementation of Hadoop's [InputFormat](https://hadoop.apache.org/docs/stable/api/org/apache/hadoop/mapred/InputFormat.html), for that. With VitessInputFormat, rows from the source table are streamed to the mapper task. Each input record has a [NullWritable](https://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/NullWritable.html) key (no key, really), and [RowWritable](https://github.com/vitessio/vitess/blob/master/java/hadoop/src/main/java/io/vitess/hadoop/RowWritable.java) as value, which is a writable implementation for the entire row's contents. Here is an example implementation of our mapper, which transforms each row into a CSV Text. diff --git a/java/jdbc/pom.xml b/java/jdbc/pom.xml index 02ada256b32..1a47533a5cb 100644 --- a/java/jdbc/pom.xml +++ b/java/jdbc/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT vitess-jdbc diff --git a/java/pom.xml b/java/pom.xml index d81deb143a1..49c3ec314a3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -11,7 +11,7 @@ io.vitess vitess-parent - 3.1.0-SNAPSHOT + 5.0-SNAPSHOT pom Vitess Java Client libraries and Hadoop support [Parent] @@ -236,7 +236,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.0 + 3.8.1 1.8 1.8 diff --git a/docker/packaging/preinstall.sh b/misc/git/hooks/golangci-lint similarity index 51% rename from docker/packaging/preinstall.sh rename to misc/git/hooks/golangci-lint index e6bfaf537a2..ea7da67e6b9 100755 --- a/docker/packaging/preinstall.sh +++ b/misc/git/hooks/golangci-lint @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright 2019 The Vitess Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,10 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -if ! /usr/bin/getent group vitess >/dev/null ; then - groupadd -r vitess -fi +# Unfortunately golangci-lint does not work well on checking just modified files. +# We will enable it for everything here, but with most of the linters disabled. +# See: https://github.com/vitessio/vitess/issues/5503 -if ! /usr/bin/getent passwd vitess >/dev/null ; then - useradd -r -g vitess vitess +GOLANGCI_LINT=$(command -v golangci-lint >/dev/null 2>&1) +if [ $? -eq 1 ]; then + echo "Downloading golangci-lint..." + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.21.0 fi + +golangci-lint run --disable=ineffassign,unused,gosimple,staticcheck,errcheck,structcheck,varcheck,deadcode diff --git a/misc/git/hooks/pylint b/misc/git/hooks/pylint index 4452647f6f9..35f04cee37c 100755 --- a/misc/git/hooks/pylint +++ b/misc/git/hooks/pylint @@ -26,7 +26,7 @@ function msg() { } PYLINT=${PYLINT:-/usr/bin/gpylint} -pylint_script=$VTTOP/tools/pylint.sh +pylint_script=$VTROOT/tools/pylint.sh # This script does not handle file names that contain spaces. pyfiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.*\.py$' | grep -v '^py/vtproto/') @@ -70,7 +70,7 @@ if [[ $? -eq 0 ]]; then do echo msg "Press enter to show the warnings for $pyfile:" - read -p " \$VTTOP/tools/pylint.sh $pyfile" + read -p " \$VTROOT/tools/pylint.sh $pyfile" $pylint_script $pyfile done read -r -p \ diff --git a/misc/git/hooks/shellcheck b/misc/git/hooks/shellcheck index 7833f2442fd..af9d8f8b37b 100755 --- a/misc/git/hooks/shellcheck +++ b/misc/git/hooks/shellcheck @@ -9,19 +9,22 @@ if [ -z "$shfiles" ] ; then exit 0 fi -# The -e SC1090,SC1091 suppressing warnings about trying to find -# files imported with "source foo.sh". We only want to lint -# the files modified as part of this current diff. -if errors=$(shellcheck -e SC1090,SC1091 "$shfiles" 2>&1); then - # No lint errors. Return early. - exit 0 -fi - if [ -z "$(command -v shellcheck)" ]; then echo "shellcheck not found, please run: brew or apt-get install shellcheck" exit 0 fi +errors= +for file in $shfiles +do + # The -e SC1090,SC1091 suppressing warnings about trying to find + # files imported with "source foo.sh". We only want to lint + # the files modified as part of this current diff. + errors+=$(shellcheck -e SC1090,SC1091 "$file" 2>&1) +done + +[ -z "$errors" ] && exit 0 + # git doesn't give us access to user input, so let's steal it. if exec < /dev/tty; then # interactive shell. Prompt the user. diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 9b7e817ed6e..ee4903bd625 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -170,6 +170,10 @@ message BinlogSource { // on_ddl specifies the action to be taken when a DDL is encountered. OnDDLAction on_ddl = 7; + + // Source is an external mysql. This attribute should be set to the username + // to use in the connection + string external_mysql = 8; } // VEventType enumerates the event types. @@ -253,6 +257,7 @@ message VEvent { FieldEvent field_event = 6; VGtid vgtid = 7; Journal journal = 8; + string dml = 9; // current_time specifies the current time to handle clock skew. int64 current_time = 20; } diff --git a/py/vtproto/automation_pb2.py b/py/vtproto/automation_pb2.py index 61f1c344ba7..eb5021f86ca 100644 --- a/py/vtproto/automation_pb2.py +++ b/py/vtproto/automation_pb2.py @@ -8,7 +8,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='automation.proto', package='automation', syntax='proto3', + serialized_options=_b('Z\'vitess.io/vitess/go/vt/proto/automation'), serialized_pb=_b('\n\x10\x61utomation.proto\x12\nautomation\"\x90\x01\n\x10\x43lusterOperation\x12\n\n\x02id\x18\x01 \x01(\t\x12/\n\x0cserial_tasks\x18\x02 \x03(\x0b\x32\x19.automation.TaskContainer\x12\x30\n\x05state\x18\x03 \x01(\x0e\x32!.automation.ClusterOperationState\x12\r\n\x05\x65rror\x18\x04 \x01(\t\"N\n\rTaskContainer\x12(\n\x0eparallel_tasks\x18\x01 \x03(\x0b\x32\x10.automation.Task\x12\x13\n\x0b\x63oncurrency\x18\x02 \x01(\x05\"\xce\x01\n\x04Task\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\nparameters\x18\x02 \x03(\x0b\x32 .automation.Task.ParametersEntry\x12\n\n\x02id\x18\x03 \x01(\t\x12$\n\x05state\x18\x04 \x01(\x0e\x32\x15.automation.TaskState\x12\x0e\n\x06output\x18\x05 \x01(\t\x12\r\n\x05\x65rror\x18\x06 \x01(\t\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb1\x01\n\x1e\x45nqueueClusterOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12N\n\nparameters\x18\x02 \x03(\x0b\x32:.automation.EnqueueClusterOperationRequest.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"-\n\x1f\x45nqueueClusterOperationResponse\x12\n\n\x02id\x18\x01 \x01(\t\"-\n\x1fGetClusterOperationStateRequest\x12\n\n\x02id\x18\x01 \x01(\t\"T\n GetClusterOperationStateResponse\x12\x30\n\x05state\x18\x01 \x01(\x0e\x32!.automation.ClusterOperationState\"/\n!GetClusterOperationDetailsRequest\x12\n\n\x02id\x18\x01 \x01(\t\"V\n\"GetClusterOperationDetailsResponse\x12\x30\n\ncluster_op\x18\x02 \x01(\x0b\x32\x1c.automation.ClusterOperation*\x9a\x01\n\x15\x43lusterOperationState\x12#\n\x1fUNKNOWN_CLUSTER_OPERATION_STATE\x10\x00\x12!\n\x1d\x43LUSTER_OPERATION_NOT_STARTED\x10\x01\x12\x1d\n\x19\x43LUSTER_OPERATION_RUNNING\x10\x02\x12\x1a\n\x16\x43LUSTER_OPERATION_DONE\x10\x03*K\n\tTaskState\x12\x16\n\x12UNKNOWN_TASK_STATE\x10\x00\x12\x0f\n\x0bNOT_STARTED\x10\x01\x12\x0b\n\x07RUNNING\x10\x02\x12\x08\n\x04\x44ONE\x10\x03\x42)Z\'vitess.io/vitess/go/vt/proto/automationb\x06proto3') ) @@ -31,23 +31,23 @@ values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_CLUSTER_OPERATION_STATE', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='CLUSTER_OPERATION_NOT_STARTED', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='CLUSTER_OPERATION_RUNNING', index=2, number=2, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='CLUSTER_OPERATION_DONE', index=3, number=3, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=966, serialized_end=1120, ) @@ -62,23 +62,23 @@ values=[ _descriptor.EnumValueDescriptor( name='UNKNOWN_TASK_STATE', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='NOT_STARTED', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='RUNNING', index=2, number=2, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='DONE', index=3, number=3, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=1122, serialized_end=1197, ) @@ -109,35 +109,35 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='serial_tasks', full_name='automation.ClusterOperation.serial_tasks', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='state', full_name='automation.ClusterOperation.state', index=2, number=3, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='error', full_name='automation.ClusterOperation.error', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -161,21 +161,21 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='concurrency', full_name='automation.TaskContainer.concurrency', index=1, number=2, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -199,21 +199,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='automation.Task.ParametersEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -236,49 +236,49 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='parameters', full_name='automation.Task.parameters', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='id', full_name='automation.Task.id', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='state', full_name='automation.Task.state', index=3, number=4, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='output', full_name='automation.Task.output', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='error', full_name='automation.Task.error', index=5, number=6, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_TASK_PARAMETERSENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -302,21 +302,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='automation.EnqueueClusterOperationRequest.ParametersEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -339,21 +339,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='parameters', full_name='automation.EnqueueClusterOperationRequest.parameters', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_ENQUEUECLUSTEROPERATIONREQUEST_PARAMETERSENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -377,14 +377,14 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -408,14 +408,14 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -439,14 +439,14 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -470,14 +470,14 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -501,14 +501,14 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -621,10 +621,7 @@ _sym_db.RegisterMessage(GetClusterOperationDetailsResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z\'vitess.io/vitess/go/vt/proto/automation')) -_TASK_PARAMETERSENTRY.has_options = True -_TASK_PARAMETERSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -_ENQUEUECLUSTEROPERATIONREQUEST_PARAMETERSENTRY.has_options = True -_ENQUEUECLUSTEROPERATIONREQUEST_PARAMETERSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +DESCRIPTOR._options = None +_TASK_PARAMETERSENTRY._options = None +_ENQUEUECLUSTEROPERATIONREQUEST_PARAMETERSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/automationservice_pb2.py b/py/vtproto/automationservice_pb2.py index 34ca83f9919..cb754003010 100644 --- a/py/vtproto/automationservice_pb2.py +++ b/py/vtproto/automationservice_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='automationservice.proto', package='automationservice', syntax='proto3', + serialized_options=_b('Z.vitess.io/vitess/go/vt/proto/automationservice'), serialized_pb=_b('\n\x17\x61utomationservice.proto\x12\x11\x61utomationservice\x1a\x10\x61utomation.proto2\x81\x02\n\nAutomation\x12t\n\x17\x45nqueueClusterOperation\x12*.automation.EnqueueClusterOperationRequest\x1a+.automation.EnqueueClusterOperationResponse\"\x00\x12}\n\x1aGetClusterOperationDetails\x12-.automation.GetClusterOperationDetailsRequest\x1a..automation.GetClusterOperationDetailsResponse\"\x00\x42\x30Z.vitess.io/vitess/go/vt/proto/automationserviceb\x06proto3') , dependencies=[automation__pb2.DESCRIPTOR,]) @@ -29,15 +29,14 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z.vitess.io/vitess/go/vt/proto/automationservice')) +DESCRIPTOR._options = None _AUTOMATION = _descriptor.ServiceDescriptor( name='Automation', full_name='automationservice.Automation', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=65, serialized_end=322, methods=[ @@ -48,7 +47,7 @@ containing_service=None, input_type=automation__pb2._ENQUEUECLUSTEROPERATIONREQUEST, output_type=automation__pb2._ENQUEUECLUSTEROPERATIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='GetClusterOperationDetails', @@ -57,7 +56,7 @@ containing_service=None, input_type=automation__pb2._GETCLUSTEROPERATIONDETAILSREQUEST, output_type=automation__pb2._GETCLUSTEROPERATIONDETAILSRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_AUTOMATION) diff --git a/py/vtproto/binlogdata_pb2.py b/py/vtproto/binlogdata_pb2.py index ae761e90e68..9f3cc1f3ea8 100644 --- a/py/vtproto/binlogdata_pb2.py +++ b/py/vtproto/binlogdata_pb2.py @@ -23,7 +23,7 @@ package='binlogdata', syntax='proto3', serialized_options=_b('Z\'vitess.io/vitess/go/vt/proto/binlogdata'), - serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\"\x9c\x01\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\x12\x39\n\x0e\x66ieldEventMode\x18\x02 \x01(\x0e\x32!.binlogdata.Filter.FieldEventMode\"6\n\x0e\x46ieldEventMode\x12\x13\n\x0f\x45RR_ON_MISMATCH\x10\x00\x12\x0f\n\x0b\x42\x45ST_EFFORT\x10\x01\"\xde\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe7\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x18\n\x10source_workflows\x18\x07 \x03(\t\"\x90\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row\"\xa7\x01\n\x15VStreamResultsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\"^\n\x16VStreamResultsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') + serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\"\x9c\x01\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\x12\x39\n\x0e\x66ieldEventMode\x18\x02 \x01(\x0e\x32!.binlogdata.Filter.FieldEventMode\"6\n\x0e\x46ieldEventMode\x12\x13\n\x0f\x45RR_ON_MISMATCH\x10\x00\x12\x0f\n\x0b\x42\x45ST_EFFORT\x10\x01\"\xf6\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\x12\x16\n\x0e\x65xternal_mysql\x18\x08 \x01(\t\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe7\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x18\n\x10source_workflows\x18\x07 \x03(\t\"\x9d\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x0b\n\x03\x64ml\x18\t \x01(\t\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row\"\xa7\x01\n\x15VStreamResultsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\"^\n\x16VStreamResultsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') , dependencies=[vtrpc__pb2.DESCRIPTOR,query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) @@ -52,8 +52,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3137, - serialized_end=3199, + serialized_start=3174, + serialized_end=3236, ) _sym_db.RegisterEnumDescriptor(_ONDDLACTION) @@ -135,8 +135,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3202, - serialized_end=3411, + serialized_start=3239, + serialized_end=3448, ) _sym_db.RegisterEnumDescriptor(_VEVENTTYPE) @@ -158,8 +158,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3413, - serialized_end=3452, + serialized_start=3450, + serialized_end=3489, ) _sym_db.RegisterEnumDescriptor(_MIGRATIONTYPE) @@ -679,6 +679,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='external_mysql', full_name='binlogdata.BinlogSource.external_mysql', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -692,7 +699,7 @@ oneofs=[ ], serialized_start=1153, - serialized_end=1375, + serialized_end=1399, ) @@ -729,8 +736,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1377, - serialized_end=1443, + serialized_start=1401, + serialized_end=1467, ) @@ -767,8 +774,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1445, - serialized_end=1519, + serialized_start=1469, + serialized_end=1543, ) @@ -805,8 +812,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1521, - serialized_end=1583, + serialized_start=1545, + serialized_end=1607, ) @@ -850,8 +857,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1585, - serialized_end=1643, + serialized_start=1609, + serialized_end=1667, ) @@ -881,8 +888,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1645, - serialized_end=1696, + serialized_start=1669, + serialized_end=1720, ) @@ -919,8 +926,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1698, - serialized_end=1746, + serialized_start=1722, + serialized_end=1770, ) @@ -992,8 +999,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1749, - serialized_end=1980, + serialized_start=1773, + serialized_end=2004, ) @@ -1061,7 +1068,14 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='current_time', full_name='binlogdata.VEvent.current_time', index=8, + name='dml', full_name='binlogdata.VEvent.dml', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='current_time', full_name='binlogdata.VEvent.current_time', index=9, number=20, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -1079,8 +1093,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1983, - serialized_end=2255, + serialized_start=2007, + serialized_end=2292, ) @@ -1138,8 +1152,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2258, - serialized_end=2457, + serialized_start=2295, + serialized_end=2494, ) @@ -1169,8 +1183,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2459, - serialized_end=2512, + serialized_start=2496, + serialized_end=2549, ) @@ -1228,8 +1242,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2515, - serialized_end=2715, + serialized_start=2552, + serialized_end=2752, ) @@ -1287,8 +1301,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2718, - serialized_end=2869, + serialized_start=2755, + serialized_end=2906, ) @@ -1339,8 +1353,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2872, - serialized_end=3039, + serialized_start=2909, + serialized_end=3076, ) @@ -1384,8 +1398,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=3041, - serialized_end=3135, + serialized_start=3078, + serialized_end=3172, ) _BINLOGTRANSACTION_STATEMENT.fields_by_name['category'].enum_type = _BINLOGTRANSACTION_STATEMENT_CATEGORY diff --git a/py/vtproto/binlogservice_pb2.py b/py/vtproto/binlogservice_pb2.py index 0dfa47c16c1..15b916eeb5f 100644 --- a/py/vtproto/binlogservice_pb2.py +++ b/py/vtproto/binlogservice_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='binlogservice.proto', package='binlogservice', syntax='proto3', + serialized_options=_b('Z*vitess.io/vitess/go/vt/proto/binlogservice'), serialized_pb=_b('\n\x13\x62inlogservice.proto\x12\rbinlogservice\x1a\x10\x62inlogdata.proto2\xc2\x01\n\x0cUpdateStream\x12[\n\x0eStreamKeyRange\x12!.binlogdata.StreamKeyRangeRequest\x1a\".binlogdata.StreamKeyRangeResponse\"\x00\x30\x01\x12U\n\x0cStreamTables\x12\x1f.binlogdata.StreamTablesRequest\x1a .binlogdata.StreamTablesResponse\"\x00\x30\x01\x42,Z*vitess.io/vitess/go/vt/proto/binlogserviceb\x06proto3') , dependencies=[binlogdata__pb2.DESCRIPTOR,]) @@ -29,15 +29,14 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z*vitess.io/vitess/go/vt/proto/binlogservice')) +DESCRIPTOR._options = None _UPDATESTREAM = _descriptor.ServiceDescriptor( name='UpdateStream', full_name='binlogservice.UpdateStream', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=57, serialized_end=251, methods=[ @@ -48,7 +47,7 @@ containing_service=None, input_type=binlogdata__pb2._STREAMKEYRANGEREQUEST, output_type=binlogdata__pb2._STREAMKEYRANGERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='StreamTables', @@ -57,7 +56,7 @@ containing_service=None, input_type=binlogdata__pb2._STREAMTABLESREQUEST, output_type=binlogdata__pb2._STREAMTABLESRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_UPDATESTREAM) diff --git a/py/vtproto/mysqlctl_pb2.py b/py/vtproto/mysqlctl_pb2.py index fc2545aaad8..5af5c27283b 100644 --- a/py/vtproto/mysqlctl_pb2.py +++ b/py/vtproto/mysqlctl_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,6 +18,7 @@ name='mysqlctl.proto', package='mysqlctl', syntax='proto3', + serialized_options=_b('Z%vitess.io/vitess/go/vt/proto/mysqlctl'), serialized_pb=_b('\n\x0emysqlctl.proto\x12\x08mysqlctl\"#\n\x0cStartRequest\x12\x13\n\x0bmysqld_args\x18\x01 \x03(\t\"\x0f\n\rStartResponse\"*\n\x0fShutdownRequest\x12\x17\n\x0fwait_for_mysqld\x18\x01 \x01(\x08\"\x12\n\x10ShutdownResponse\"\x18\n\x16RunMysqlUpgradeRequest\"\x19\n\x17RunMysqlUpgradeResponse\"\x15\n\x13ReinitConfigRequest\"\x16\n\x14ReinitConfigResponse\"\x16\n\x14RefreshConfigRequest\"\x17\n\x15RefreshConfigResponse2\x8a\x03\n\x08MysqlCtl\x12:\n\x05Start\x12\x16.mysqlctl.StartRequest\x1a\x17.mysqlctl.StartResponse\"\x00\x12\x43\n\x08Shutdown\x12\x19.mysqlctl.ShutdownRequest\x1a\x1a.mysqlctl.ShutdownResponse\"\x00\x12X\n\x0fRunMysqlUpgrade\x12 .mysqlctl.RunMysqlUpgradeRequest\x1a!.mysqlctl.RunMysqlUpgradeResponse\"\x00\x12O\n\x0cReinitConfig\x12\x1d.mysqlctl.ReinitConfigRequest\x1a\x1e.mysqlctl.ReinitConfigResponse\"\x00\x12R\n\rRefreshConfig\x12\x1e.mysqlctl.RefreshConfigRequest\x1a\x1f.mysqlctl.RefreshConfigResponse\"\x00\x42\'Z%vitess.io/vitess/go/vt/proto/mysqlctlb\x06proto3') ) @@ -38,14 +38,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -69,7 +69,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -93,14 +93,14 @@ has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -124,7 +124,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -148,7 +148,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -172,7 +172,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -196,7 +196,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -220,7 +220,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -244,7 +244,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -268,7 +268,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -361,15 +361,14 @@ _sym_db.RegisterMessage(RefreshConfigResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z%vitess.io/vitess/go/vt/proto/mysqlctl')) +DESCRIPTOR._options = None _MYSQLCTL = _descriptor.ServiceDescriptor( name='MysqlCtl', full_name='mysqlctl.MysqlCtl', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=296, serialized_end=690, methods=[ @@ -380,7 +379,7 @@ containing_service=None, input_type=_STARTREQUEST, output_type=_STARTRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='Shutdown', @@ -389,7 +388,7 @@ containing_service=None, input_type=_SHUTDOWNREQUEST, output_type=_SHUTDOWNRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='RunMysqlUpgrade', @@ -398,7 +397,7 @@ containing_service=None, input_type=_RUNMYSQLUPGRADEREQUEST, output_type=_RUNMYSQLUPGRADERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='ReinitConfig', @@ -407,7 +406,7 @@ containing_service=None, input_type=_REINITCONFIGREQUEST, output_type=_REINITCONFIGRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='RefreshConfig', @@ -416,7 +415,7 @@ containing_service=None, input_type=_REFRESHCONFIGREQUEST, output_type=_REFRESHCONFIGRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_MYSQLCTL) diff --git a/py/vtproto/replicationdata_pb2.py b/py/vtproto/replicationdata_pb2.py index e76d4591683..972a3970aee 100644 --- a/py/vtproto/replicationdata_pb2.py +++ b/py/vtproto/replicationdata_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,6 +18,7 @@ name='replicationdata.proto', package='replicationdata', syntax='proto3', + serialized_options=_b('Z,vitess.io/vitess/go/vt/proto/replicationdata'), serialized_pb=_b('\n\x15replicationdata.proto\x12\x0freplicationdata\"\xb6\x01\n\x06Status\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x18\n\x10slave_io_running\x18\x02 \x01(\x08\x12\x19\n\x11slave_sql_running\x18\x03 \x01(\x08\x12\x1d\n\x15seconds_behind_master\x18\x04 \x01(\r\x12\x13\n\x0bmaster_host\x18\x05 \x01(\t\x12\x13\n\x0bmaster_port\x18\x06 \x01(\x05\x12\x1c\n\x14master_connect_retry\x18\x07 \x01(\x05\x42.Z,vitess.io/vitess/go/vt/proto/replicationdatab\x06proto3') ) @@ -38,56 +38,56 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='slave_io_running', full_name='replicationdata.Status.slave_io_running', index=1, number=2, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='slave_sql_running', full_name='replicationdata.Status.slave_sql_running', index=2, number=3, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='seconds_behind_master', full_name='replicationdata.Status.seconds_behind_master', index=3, number=4, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='master_host', full_name='replicationdata.Status.master_host', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='master_port', full_name='replicationdata.Status.master_port', index=5, number=6, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='master_connect_retry', full_name='replicationdata.Status.master_connect_retry', index=6, number=7, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -108,6 +108,5 @@ _sym_db.RegisterMessage(Status) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z,vitess.io/vitess/go/vt/proto/replicationdata')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/tableacl_pb2.py b/py/vtproto/tableacl_pb2.py index cfdd8d0e63c..793bfdd5886 100644 --- a/py/vtproto/tableacl_pb2.py +++ b/py/vtproto/tableacl_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,6 +18,7 @@ name='tableacl.proto', package='tableacl', syntax='proto3', + serialized_options=_b('Z%vitess.io/vitess/go/vt/proto/tableacl'), serialized_pb=_b('\n\x0etableacl.proto\x12\x08tableacl\"q\n\x0eTableGroupSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x17table_names_or_prefixes\x18\x02 \x03(\t\x12\x0f\n\x07readers\x18\x03 \x03(\t\x12\x0f\n\x07writers\x18\x04 \x03(\t\x12\x0e\n\x06\x61\x64mins\x18\x05 \x03(\t\"8\n\x06\x43onfig\x12.\n\x0ctable_groups\x18\x01 \x03(\x0b\x32\x18.tableacl.TableGroupSpecB\'Z%vitess.io/vitess/go/vt/proto/tableaclb\x06proto3') ) @@ -38,42 +38,42 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='table_names_or_prefixes', full_name='tableacl.TableGroupSpec.table_names_or_prefixes', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='readers', full_name='tableacl.TableGroupSpec.readers', index=2, number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='writers', full_name='tableacl.TableGroupSpec.writers', index=3, number=4, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='admins', full_name='tableacl.TableGroupSpec.admins', index=4, number=5, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -97,14 +97,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -134,6 +134,5 @@ _sym_db.RegisterMessage(Config) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z%vitess.io/vitess/go/vt/proto/tableacl')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/throttlerdata_pb2.py b/py/vtproto/throttlerdata_pb2.py index 5706e958988..c531a2912a3 100644 --- a/py/vtproto/throttlerdata_pb2.py +++ b/py/vtproto/throttlerdata_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,6 +18,7 @@ name='throttlerdata.proto', package='throttlerdata', syntax='proto3', + serialized_options=_b('Z*vitess.io/vitess/go/vt/proto/throttlerdata'), serialized_pb=_b('\n\x13throttlerdata.proto\x12\rthrottlerdata\"\x11\n\x0fMaxRatesRequest\"{\n\x10MaxRatesResponse\x12\x39\n\x05rates\x18\x01 \x03(\x0b\x32*.throttlerdata.MaxRatesResponse.RatesEntry\x1a,\n\nRatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"!\n\x11SetMaxRateRequest\x12\x0c\n\x04rate\x18\x01 \x01(\x03\"#\n\x12SetMaxRateResponse\x12\r\n\x05names\x18\x01 \x03(\t\"\xe8\x03\n\rConfiguration\x12\"\n\x1atarget_replication_lag_sec\x18\x01 \x01(\x03\x12\x1f\n\x17max_replication_lag_sec\x18\x02 \x01(\x03\x12\x14\n\x0cinitial_rate\x18\x03 \x01(\x03\x12\x14\n\x0cmax_increase\x18\x04 \x01(\x01\x12\x1a\n\x12\x65mergency_decrease\x18\x05 \x01(\x01\x12*\n\"min_duration_between_increases_sec\x18\x06 \x01(\x03\x12*\n\"max_duration_between_increases_sec\x18\x07 \x01(\x03\x12*\n\"min_duration_between_decreases_sec\x18\x08 \x01(\x03\x12!\n\x19spread_backlog_across_sec\x18\t \x01(\x03\x12!\n\x19ignore_n_slowest_replicas\x18\n \x01(\x05\x12 \n\x18ignore_n_slowest_rdonlys\x18\x0b \x01(\x05\x12\x1e\n\x16\x61ge_bad_rate_after_sec\x18\x0c \x01(\x03\x12\x19\n\x11\x62\x61\x64_rate_increase\x18\r \x01(\x01\x12#\n\x1bmax_rate_approach_threshold\x18\x0e \x01(\x01\"1\n\x17GetConfigurationRequest\x12\x16\n\x0ethrottler_name\x18\x01 \x01(\t\"\xc4\x01\n\x18GetConfigurationResponse\x12S\n\x0e\x63onfigurations\x18\x01 \x03(\x0b\x32;.throttlerdata.GetConfigurationResponse.ConfigurationsEntry\x1aS\n\x13\x43onfigurationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.throttlerdata.Configuration:\x02\x38\x01\"\x83\x01\n\x1aUpdateConfigurationRequest\x12\x16\n\x0ethrottler_name\x18\x01 \x01(\t\x12\x33\n\rconfiguration\x18\x02 \x01(\x0b\x32\x1c.throttlerdata.Configuration\x12\x18\n\x10\x63opy_zero_values\x18\x03 \x01(\x08\",\n\x1bUpdateConfigurationResponse\x12\r\n\x05names\x18\x01 \x03(\t\"3\n\x19ResetConfigurationRequest\x12\x16\n\x0ethrottler_name\x18\x01 \x01(\t\"+\n\x1aResetConfigurationResponse\x12\r\n\x05names\x18\x01 \x03(\tB,Z*vitess.io/vitess/go/vt/proto/throttlerdatab\x06proto3') ) @@ -38,7 +38,7 @@ nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -62,21 +62,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='throttlerdata.MaxRatesResponse.RatesEntry.value', index=1, number=2, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -99,14 +99,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_MAXRATESRESPONSE_RATESENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -130,14 +130,14 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -161,14 +161,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -192,105 +192,105 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='max_replication_lag_sec', full_name='throttlerdata.Configuration.max_replication_lag_sec', index=1, number=2, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='initial_rate', full_name='throttlerdata.Configuration.initial_rate', index=2, number=3, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='max_increase', full_name='throttlerdata.Configuration.max_increase', index=3, number=4, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='emergency_decrease', full_name='throttlerdata.Configuration.emergency_decrease', index=4, number=5, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='min_duration_between_increases_sec', full_name='throttlerdata.Configuration.min_duration_between_increases_sec', index=5, number=6, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='max_duration_between_increases_sec', full_name='throttlerdata.Configuration.max_duration_between_increases_sec', index=6, number=7, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='min_duration_between_decreases_sec', full_name='throttlerdata.Configuration.min_duration_between_decreases_sec', index=7, number=8, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='spread_backlog_across_sec', full_name='throttlerdata.Configuration.spread_backlog_across_sec', index=8, number=9, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='ignore_n_slowest_replicas', full_name='throttlerdata.Configuration.ignore_n_slowest_replicas', index=9, number=10, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='ignore_n_slowest_rdonlys', full_name='throttlerdata.Configuration.ignore_n_slowest_rdonlys', index=10, number=11, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='age_bad_rate_after_sec', full_name='throttlerdata.Configuration.age_bad_rate_after_sec', index=11, number=12, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='bad_rate_increase', full_name='throttlerdata.Configuration.bad_rate_increase', index=12, number=13, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='max_rate_approach_threshold', full_name='throttlerdata.Configuration.max_rate_approach_threshold', index=13, number=14, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -314,14 +314,14 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -345,21 +345,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='throttlerdata.GetConfigurationResponse.ConfigurationsEntry.value', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -382,14 +382,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_GETCONFIGURATIONRESPONSE_CONFIGURATIONSENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -413,28 +413,28 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='configuration', full_name='throttlerdata.UpdateConfigurationRequest.configuration', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='copy_zero_values', full_name='throttlerdata.UpdateConfigurationRequest.copy_zero_values', index=2, number=3, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -458,14 +458,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -489,14 +489,14 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -520,14 +520,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -650,10 +650,7 @@ _sym_db.RegisterMessage(ResetConfigurationResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z*vitess.io/vitess/go/vt/proto/throttlerdata')) -_MAXRATESRESPONSE_RATESENTRY.has_options = True -_MAXRATESRESPONSE_RATESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -_GETCONFIGURATIONRESPONSE_CONFIGURATIONSENTRY.has_options = True -_GETCONFIGURATIONRESPONSE_CONFIGURATIONSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +DESCRIPTOR._options = None +_MAXRATESRESPONSE_RATESENTRY._options = None +_GETCONFIGURATIONRESPONSE_CONFIGURATIONSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/throttlerservice_pb2.py b/py/vtproto/throttlerservice_pb2.py index f4bdd95acb6..409f04c7ad9 100644 --- a/py/vtproto/throttlerservice_pb2.py +++ b/py/vtproto/throttlerservice_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='throttlerservice.proto', package='throttlerservice', syntax='proto3', + serialized_options=_b('Z-vitess.io/vitess/go/vt/proto/throttlerservice'), serialized_pb=_b('\n\x16throttlerservice.proto\x12\x10throttlerservice\x1a\x13throttlerdata.proto2\xf3\x03\n\tThrottler\x12M\n\x08MaxRates\x12\x1e.throttlerdata.MaxRatesRequest\x1a\x1f.throttlerdata.MaxRatesResponse\"\x00\x12S\n\nSetMaxRate\x12 .throttlerdata.SetMaxRateRequest\x1a!.throttlerdata.SetMaxRateResponse\"\x00\x12\x65\n\x10GetConfiguration\x12&.throttlerdata.GetConfigurationRequest\x1a\'.throttlerdata.GetConfigurationResponse\"\x00\x12n\n\x13UpdateConfiguration\x12).throttlerdata.UpdateConfigurationRequest\x1a*.throttlerdata.UpdateConfigurationResponse\"\x00\x12k\n\x12ResetConfiguration\x12(.throttlerdata.ResetConfigurationRequest\x1a).throttlerdata.ResetConfigurationResponse\"\x00\x42/Z-vitess.io/vitess/go/vt/proto/throttlerserviceb\x06proto3') , dependencies=[throttlerdata__pb2.DESCRIPTOR,]) @@ -29,15 +29,14 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z-vitess.io/vitess/go/vt/proto/throttlerservice')) +DESCRIPTOR._options = None _THROTTLER = _descriptor.ServiceDescriptor( name='Throttler', full_name='throttlerservice.Throttler', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=66, serialized_end=565, methods=[ @@ -48,7 +47,7 @@ containing_service=None, input_type=throttlerdata__pb2._MAXRATESREQUEST, output_type=throttlerdata__pb2._MAXRATESRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='SetMaxRate', @@ -57,7 +56,7 @@ containing_service=None, input_type=throttlerdata__pb2._SETMAXRATEREQUEST, output_type=throttlerdata__pb2._SETMAXRATERESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='GetConfiguration', @@ -66,7 +65,7 @@ containing_service=None, input_type=throttlerdata__pb2._GETCONFIGURATIONREQUEST, output_type=throttlerdata__pb2._GETCONFIGURATIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='UpdateConfiguration', @@ -75,7 +74,7 @@ containing_service=None, input_type=throttlerdata__pb2._UPDATECONFIGURATIONREQUEST, output_type=throttlerdata__pb2._UPDATECONFIGURATIONRESPONSE, - options=None, + serialized_options=None, ), _descriptor.MethodDescriptor( name='ResetConfiguration', @@ -84,7 +83,7 @@ containing_service=None, input_type=throttlerdata__pb2._RESETCONFIGURATIONREQUEST, output_type=throttlerdata__pb2._RESETCONFIGURATIONRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_THROTTLER) diff --git a/py/vtproto/vtctldata_pb2.py b/py/vtproto/vtctldata_pb2.py index 766a844c18b..4e2a0c5c11e 100644 --- a/py/vtproto/vtctldata_pb2.py +++ b/py/vtproto/vtctldata_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='vtctldata.proto', package='vtctldata', syntax='proto3', + serialized_options=_b('Z&vitess.io/vitess/go/vt/proto/vtctldata'), serialized_pb=_b('\n\x0fvtctldata.proto\x12\tvtctldata\x1a\rlogutil.proto\"B\n\x1a\x45xecuteVtctlCommandRequest\x12\x0c\n\x04\x61rgs\x18\x01 \x03(\t\x12\x16\n\x0e\x61\x63tion_timeout\x18\x02 \x01(\x03\"<\n\x1b\x45xecuteVtctlCommandResponse\x12\x1d\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x0e.logutil.EventB(Z&vitess.io/vitess/go/vt/proto/vtctldatab\x06proto3') , dependencies=[logutil__pb2.DESCRIPTOR,]) @@ -40,21 +40,21 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='action_timeout', full_name='vtctldata.ExecuteVtctlCommandRequest.action_timeout', index=1, number=2, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -78,14 +78,14 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -115,6 +115,5 @@ _sym_db.RegisterMessage(ExecuteVtctlCommandResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z&vitess.io/vitess/go/vt/proto/vtctldata')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/vtctlservice_pb2.py b/py/vtproto/vtctlservice_pb2.py index 431d0ac193f..5aa94a82ba0 100644 --- a/py/vtproto/vtctlservice_pb2.py +++ b/py/vtproto/vtctlservice_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='vtctlservice.proto', package='vtctlservice', syntax='proto3', + serialized_options=_b('Z)vitess.io/vitess/go/vt/proto/vtctlservice'), serialized_pb=_b('\n\x12vtctlservice.proto\x12\x0cvtctlservice\x1a\x0fvtctldata.proto2q\n\x05Vtctl\x12h\n\x13\x45xecuteVtctlCommand\x12%.vtctldata.ExecuteVtctlCommandRequest\x1a&.vtctldata.ExecuteVtctlCommandResponse\"\x00\x30\x01\x42+Z)vitess.io/vitess/go/vt/proto/vtctlserviceb\x06proto3') , dependencies=[vtctldata__pb2.DESCRIPTOR,]) @@ -29,15 +29,14 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z)vitess.io/vitess/go/vt/proto/vtctlservice')) +DESCRIPTOR._options = None _VTCTL = _descriptor.ServiceDescriptor( name='Vtctl', full_name='vtctlservice.Vtctl', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=53, serialized_end=166, methods=[ @@ -48,7 +47,7 @@ containing_service=None, input_type=vtctldata__pb2._EXECUTEVTCTLCOMMANDREQUEST, output_type=vtctldata__pb2._EXECUTEVTCTLCOMMANDRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_VTCTL) diff --git a/py/vtproto/vttest_pb2.py b/py/vtproto/vttest_pb2.py index 5e20f3c45f1..1c414f959bd 100644 --- a/py/vtproto/vttest_pb2.py +++ b/py/vtproto/vttest_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,6 +18,7 @@ name='vttest.proto', package='vttest', syntax='proto3', + serialized_options=_b('Z#vitess.io/vitess/go/vt/proto/vttest'), serialized_pb=_b('\n\x0cvttest.proto\x12\x06vttest\"/\n\x05Shard\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x10\x64\x62_name_override\x18\x02 \x01(\t\"\xb5\x01\n\x08Keyspace\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1d\n\x06shards\x18\x02 \x03(\x0b\x32\r.vttest.Shard\x12\x1c\n\x14sharding_column_name\x18\x03 \x01(\t\x12\x1c\n\x14sharding_column_type\x18\x04 \x01(\t\x12\x13\n\x0bserved_from\x18\x05 \x01(\t\x12\x15\n\rreplica_count\x18\x06 \x01(\x05\x12\x14\n\x0crdonly_count\x18\x07 \x01(\x05\"D\n\x0eVTTestTopology\x12#\n\tkeyspaces\x18\x01 \x03(\x0b\x32\x10.vttest.Keyspace\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\tB%Z#vitess.io/vitess/go/vt/proto/vttestb\x06proto3') ) @@ -38,21 +38,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='db_name_override', full_name='vttest.Shard.db_name_override', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -76,56 +76,56 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='shards', full_name='vttest.Keyspace.shards', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='sharding_column_name', full_name='vttest.Keyspace.sharding_column_name', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='sharding_column_type', full_name='vttest.Keyspace.sharding_column_type', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='served_from', full_name='vttest.Keyspace.served_from', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='replica_count', full_name='vttest.Keyspace.replica_count', index=5, number=6, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='rdonly_count', full_name='vttest.Keyspace.rdonly_count', index=6, number=7, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -149,21 +149,21 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='cells', full_name='vttest.VTTestTopology.cells', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -202,6 +202,5 @@ _sym_db.RegisterMessage(VTTestTopology) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z#vitess.io/vitess/go/vt/proto/vttest')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/vtworkerdata_pb2.py b/py/vtproto/vtworkerdata_pb2.py index a874b812746..f319647d24b 100644 --- a/py/vtproto/vtworkerdata_pb2.py +++ b/py/vtproto/vtworkerdata_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='vtworkerdata.proto', package='vtworkerdata', syntax='proto3', + serialized_options=_b('Z)vitess.io/vitess/go/vt/proto/vtworkerdata'), serialized_pb=_b('\n\x12vtworkerdata.proto\x12\x0cvtworkerdata\x1a\rlogutil.proto\"-\n\x1d\x45xecuteVtworkerCommandRequest\x12\x0c\n\x04\x61rgs\x18\x01 \x03(\t\"?\n\x1e\x45xecuteVtworkerCommandResponse\x12\x1d\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x0e.logutil.EventB+Z)vitess.io/vitess/go/vt/proto/vtworkerdatab\x06proto3') , dependencies=[logutil__pb2.DESCRIPTOR,]) @@ -40,14 +40,14 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -71,14 +71,14 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -108,6 +108,5 @@ _sym_db.RegisterMessage(ExecuteVtworkerCommandResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z)vitess.io/vitess/go/vt/proto/vtworkerdata')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/vtworkerservice_pb2.py b/py/vtproto/vtworkerservice_pb2.py index ee25dd13b62..9c7ad0640ab 100644 --- a/py/vtproto/vtworkerservice_pb2.py +++ b/py/vtproto/vtworkerservice_pb2.py @@ -7,7 +7,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='vtworkerservice.proto', package='vtworkerservice', syntax='proto3', + serialized_options=_b('Z,vitess.io/vitess/go/vt/proto/vtworkerservice'), serialized_pb=_b('\n\x15vtworkerservice.proto\x12\x0fvtworkerservice\x1a\x12vtworkerdata.proto2\x83\x01\n\x08Vtworker\x12w\n\x16\x45xecuteVtworkerCommand\x12+.vtworkerdata.ExecuteVtworkerCommandRequest\x1a,.vtworkerdata.ExecuteVtworkerCommandResponse\"\x00\x30\x01\x42.Z,vitess.io/vitess/go/vt/proto/vtworkerserviceb\x06proto3') , dependencies=[vtworkerdata__pb2.DESCRIPTOR,]) @@ -29,15 +29,14 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z,vitess.io/vitess/go/vt/proto/vtworkerservice')) +DESCRIPTOR._options = None _VTWORKER = _descriptor.ServiceDescriptor( name='Vtworker', full_name='vtworkerservice.Vtworker', file=DESCRIPTOR, index=0, - options=None, + serialized_options=None, serialized_start=63, serialized_end=194, methods=[ @@ -48,7 +47,7 @@ containing_service=None, input_type=vtworkerdata__pb2._EXECUTEVTWORKERCOMMANDREQUEST, output_type=vtworkerdata__pb2._EXECUTEVTWORKERCOMMANDRESPONSE, - options=None, + serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_VTWORKER) diff --git a/py/vtproto/workflow_pb2.py b/py/vtproto/workflow_pb2.py index 5d856a412f0..c624dfe8fc4 100644 --- a/py/vtproto/workflow_pb2.py +++ b/py/vtproto/workflow_pb2.py @@ -8,7 +8,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='workflow.proto', package='workflow', syntax='proto3', + serialized_options=_b('Z%vitess.io/vitess/go/vt/proto/workflow'), serialized_pb=_b('\n\x0eworkflow.proto\x12\x08workflow\"\xbc\x01\n\x08Workflow\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x61\x63tory_name\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12&\n\x05state\x18\x04 \x01(\x0e\x32\x17.workflow.WorkflowState\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\x12\r\n\x05\x65rror\x18\x06 \x01(\t\x12\x12\n\nstart_time\x18\x07 \x01(\x03\x12\x10\n\x08\x65nd_time\x18\x08 \x01(\x03\x12\x13\n\x0b\x63reate_time\x18\t \x01(\x03\"\x8f\x02\n\x12WorkflowCheckpoint\x12\x14\n\x0c\x63ode_version\x18\x01 \x01(\x05\x12\x36\n\x05tasks\x18\x02 \x03(\x0b\x32\'.workflow.WorkflowCheckpoint.TasksEntry\x12<\n\x08settings\x18\x03 \x03(\x0b\x32*.workflow.WorkflowCheckpoint.SettingsEntry\x1a<\n\nTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1d\n\x05value\x18\x02 \x01(\x0b\x32\x0e.workflow.Task:\x02\x38\x01\x1a/\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xac\x01\n\x04Task\x12\n\n\x02id\x18\x01 \x01(\t\x12\"\n\x05state\x18\x02 \x01(\x0e\x32\x13.workflow.TaskState\x12\x32\n\nattributes\x18\x03 \x03(\x0b\x32\x1e.workflow.Task.AttributesEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x1a\x31\n\x0f\x41ttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01*6\n\rWorkflowState\x12\x0e\n\nNotStarted\x10\x00\x12\x0b\n\x07Running\x10\x01\x12\x08\n\x04\x44one\x10\x02*>\n\tTaskState\x12\x12\n\x0eTaskNotStarted\x10\x00\x12\x0f\n\x0bTaskRunning\x10\x01\x12\x0c\n\x08TaskDone\x10\x02\x42\'Z%vitess.io/vitess/go/vt/proto/workflowb\x06proto3') ) @@ -31,19 +31,19 @@ values=[ _descriptor.EnumValueDescriptor( name='NotStarted', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='Running', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='Done', index=2, number=2, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=668, serialized_end=722, ) @@ -58,19 +58,19 @@ values=[ _descriptor.EnumValueDescriptor( name='TaskNotStarted', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='TaskRunning', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='TaskDone', index=2, number=2, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=724, serialized_end=786, ) @@ -99,70 +99,70 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='factory_name', full_name='workflow.Workflow.factory_name', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='name', full_name='workflow.Workflow.name', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='state', full_name='workflow.Workflow.state', index=3, number=4, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='data', full_name='workflow.Workflow.data', index=4, number=5, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='error', full_name='workflow.Workflow.error', index=5, number=6, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='start_time', full_name='workflow.Workflow.start_time', index=6, number=7, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='end_time', full_name='workflow.Workflow.end_time', index=7, number=8, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='create_time', full_name='workflow.Workflow.create_time', index=8, number=9, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -186,21 +186,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='workflow.WorkflowCheckpoint.TasksEntry.value', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -223,21 +223,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='workflow.WorkflowCheckpoint.SettingsEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -260,28 +260,28 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='tasks', full_name='workflow.WorkflowCheckpoint.tasks', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='settings', full_name='workflow.WorkflowCheckpoint.settings', index=2, number=3, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_WORKFLOWCHECKPOINT_TASKSENTRY, _WORKFLOWCHECKPOINT_SETTINGSENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -305,21 +305,21 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='workflow.Task.AttributesEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -342,35 +342,35 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='state', full_name='workflow.Task.state', index=1, number=2, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='attributes', full_name='workflow.Task.attributes', index=2, number=3, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='error', full_name='workflow.Task.error', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[_TASK_ATTRIBUTESENTRY, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -442,12 +442,8 @@ _sym_db.RegisterMessage(Task.AttributesEntry) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('Z%vitess.io/vitess/go/vt/proto/workflow')) -_WORKFLOWCHECKPOINT_TASKSENTRY.has_options = True -_WORKFLOWCHECKPOINT_TASKSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -_WORKFLOWCHECKPOINT_SETTINGSENTRY.has_options = True -_WORKFLOWCHECKPOINT_SETTINGSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -_TASK_ATTRIBUTESENTRY.has_options = True -_TASK_ATTRIBUTESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +DESCRIPTOR._options = None +_WORKFLOWCHECKPOINT_TASKSENTRY._options = None +_WORKFLOWCHECKPOINT_SETTINGSENTRY._options = None +_TASK_ATTRIBUTESENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/py/vttest/local_database.py b/py/vttest/local_database.py index fb3adeba309..458ea5c2c62 100644 --- a/py/vttest/local_database.py +++ b/py/vttest/local_database.py @@ -37,7 +37,8 @@ def __init__(self, extra_my_cnf=None, web_dir2=None, snapshot_file=None, - charset='utf8'): + charset='utf8', + mysql_server_bind_address=None): """Initializes an object of this class. Args: @@ -59,6 +60,7 @@ def __init__(self, flag in run_local_database.py snapshot_file: A MySQL DB snapshot file. charset: MySQL charset. + mysql_server_bind_address: MySQL server bind address. """ self.topology = topology @@ -71,6 +73,7 @@ def __init__(self, self.web_dir2 = web_dir2 self.snapshot_file = snapshot_file self.charset = charset + self.mysql_server_bind_address = mysql_server_bind_address def setup(self): """Create a MySQL instance and all Vitess processes.""" @@ -92,7 +95,8 @@ def setup(self): vt_processes.start_vt_processes(self.directory, self.topology, self.mysql_db, self.schema_dir, charset=self.charset, web_dir=self.web_dir, - web_dir2=self.web_dir2) + web_dir2=self.web_dir2, + mysql_server_bind_address=self.mysql_server_bind_address) def teardown(self): """Kill all Vitess processes and wait for them to end. diff --git a/py/vttest/mysql_db_mysqlctl.py b/py/vttest/mysql_db_mysqlctl.py index f79f5c6504f..c1ecc8d049f 100644 --- a/py/vttest/mysql_db_mysqlctl.py +++ b/py/vttest/mysql_db_mysqlctl.py @@ -40,7 +40,7 @@ def setup(self): '-mysql_port', str(self._port), 'init', '-init_db_sql_file', - os.path.join(os.environ['VTTOP'], 'config/init_db.sql'), + os.path.join(os.environ['VTROOT'], 'config/init_db.sql'), ] env = os.environ env['VTDATAROOT'] = self._directory diff --git a/py/vttest/mysql_flavor.py b/py/vttest/mysql_flavor.py index 26e8aaf3d49..c28bf979e02 100644 --- a/py/vttest/mysql_flavor.py +++ b/py/vttest/mysql_flavor.py @@ -23,17 +23,15 @@ import sys -# For now, vttop is only used in this module. If other people +# For now, vtroot is only used in this module. If other people # need this, we should move it to environment. -if "VTTOP" not in os.environ: +if "VTROOT" not in os.environ: sys.stderr.write( "ERROR: Vitess environment not set up. " 'Please run "source dev.env" first.\n') sys.exit(1) -# vttop is the toplevel of the vitess source tree -vttop = os.environ["VTTOP"] - +vtroot = os.environ["VTROOT"] class MysqlFlavor(object): """Base class with default SQL statements.""" @@ -48,8 +46,7 @@ class MariaDB(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mariadb100.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), ] return ":".join(files) @@ -58,8 +55,7 @@ class MariaDB103(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mariadb103.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), ] return ":".join(files) @@ -68,8 +64,7 @@ class MySQL56(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mysql56.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), ] return ":".join(files) @@ -78,8 +73,7 @@ class MySQL80(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mysql80.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), ] return ":".join(files) diff --git a/py/vttest/run_local_database.py b/py/vttest/run_local_database.py index 8301a4e2b42..05faf89eb7b 100755 --- a/py/vttest/run_local_database.py +++ b/py/vttest/run_local_database.py @@ -98,7 +98,7 @@ def main(cmdline_options): init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size init_data_opts.null_probability = cmdline_options.null_probability - extra_my_cnf = os.path.join(os.environ['VTTOP'], 'config/mycnf/vtcombo.cnf') + extra_my_cnf = '' if cmdline_options.extra_my_cnf: extra_my_cnf += ':' + cmdline_options.extra_my_cnf @@ -112,7 +112,8 @@ def main(cmdline_options): default_schema_dir=cmdline_options.default_schema_dir, extra_my_cnf=extra_my_cnf, charset=cmdline_options.charset, - snapshot_file=cmdline_options.snapshot_file) as local_db: + snapshot_file=cmdline_options.snapshot_file, + mysql_server_bind_address=cmdline_options.mysql_server_bind_address) as local_db: print json.dumps(local_db.config()) sys.stdout.flush() try: @@ -189,6 +190,9 @@ def main(cmdline_options): parser.add_option( '-f', '--extra_my_cnf', help='extra files to add to the config, separated by ":"') + parser.add_option( + '--mysql_server_bind_address', + help='mysql server bind address ":"') parser.add_option( '-v', '--verbose', action='store_true', help='Display extra error messages.') diff --git a/py/vttest/vt_processes.py b/py/vttest/vt_processes.py index 14de9fb347e..0df9a4b7a6a 100644 --- a/py/vttest/vt_processes.py +++ b/py/vttest/vt_processes.py @@ -137,7 +137,7 @@ class VtcomboProcess(VtProcess): ] def __init__(self, directory, topology, mysql_db, schema_dir, charset, - web_dir=None, web_dir2=None): + web_dir=None, web_dir2=None,mysql_server_bind_address=None): VtProcess.__init__(self, 'vtcombo-%s' % os.environ['USER'], directory, environment.vtcombo_binary, port_name='vtcombo') self.extraparams = [ @@ -164,17 +164,21 @@ def __init__(self, directory, topology, mysql_db, schema_dir, charset, ['-db_host', mysql_db.hostname(), '-db_port', str(mysql_db.port())]) self.vtcombo_mysql_port = environment.get_port('vtcombo_mysql_port') + if mysql_server_bind_address: + # Binding to 0.0.0.0 instead of localhost makes it possible to connect to vtgate from outside a docker container + self.extraparams.extend(['-mysql_server_bind_address', mysql_server_bind_address]) + else: + self.extraparams.extend(['-mysql_server_bind_address', 'localhost']) self.extraparams.extend( ['-mysql_auth_server_impl', 'none', - '-mysql_server_port', str(self.vtcombo_mysql_port), - '-mysql_server_bind_address', 'localhost']) + '-mysql_server_port', str(self.vtcombo_mysql_port)]) vtcombo_process = None def start_vt_processes(directory, topology, mysql_db, schema_dir, - charset='utf8', web_dir=None, web_dir2=None): + charset='utf8', web_dir=None, web_dir2=None, mysql_server_bind_address=None): """Start the vt processes. Args: @@ -185,13 +189,15 @@ def start_vt_processes(directory, topology, mysql_db, schema_dir, charset: the character set for the database connections. web_dir: contains the web app for vtctld side of vtcombo. web_dir2: contains the web app for vtctld side of vtcombo. + mysql_server_bind_address: MySQL server bind address for vtcombo. """ global vtcombo_process logging.info('start_vt_processes(directory=%s,vtcombo_binary=%s)', directory, environment.vtcombo_binary) vtcombo_process = VtcomboProcess(directory, topology, mysql_db, schema_dir, - charset, web_dir=web_dir, web_dir2=web_dir2) + charset, web_dir=web_dir, web_dir2=web_dir2, + mysql_server_bind_address=mysql_server_bind_address) vtcombo_process.wait_start() diff --git a/test.go b/test.go index 73b75e9df30..846cb0c8755 100755 --- a/test.go +++ b/test.go @@ -27,7 +27,7 @@ run against a given flavor, it may take some time for the corresponding bootstrap image (vitess/bootstrap:) to be downloaded. It is meant to be run from the Vitess root, like so: - ~/src/vitess.io/vitess$ go run test.go [args] + $ go run test.go [args] For a list of options, run: $ go run test.go --help @@ -195,6 +195,7 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) { // Also try to make them use different port ranges // to mitigate failures due to zombie processes. cmd.Env = updateEnv(os.Environ(), map[string]string{ + "VTROOT": "/vt/src/vitess.io/vitess", "VTDATAROOT": dataDir, "VTPORTSTART": strconv.FormatInt(int64(getPortStart(100)), 10), }) @@ -370,7 +371,7 @@ func main() { } tests = dup - vtTop := "." + vtRoot := "." tmpDir := "" if *docker { // Copy working repo to tmpDir. @@ -387,7 +388,7 @@ func main() { if out, err := exec.Command("chmod", "-R", "go=u", tmpDir).CombinedOutput(); err != nil { log.Printf("Can't set permissions on temp dir %v: %v: %s", tmpDir, err, out) } - vtTop = tmpDir + vtRoot = tmpDir } else { // Since we're sharing the working dir, do the build once for all tests. log.Printf("Running make build...") @@ -473,7 +474,7 @@ func main() { // Run the test. start := time.Now() - output, err := test.run(vtTop, dataDir) + output, err := test.run(vtRoot, dataDir) duration := time.Since(start) // Save/print test output. diff --git a/test/backup.py b/test/backup.py index 823e2a2ba2a..51b7a94eef4 100755 --- a/test/backup.py +++ b/test/backup.py @@ -95,7 +95,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/backup_only.py b/test/backup_only.py index dd9b547dacf..04f2db4dfdc 100755 --- a/test/backup_only.py +++ b/test/backup_only.py @@ -95,7 +95,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/backup_transform.py b/test/backup_transform.py index b0e665b6fa8..00f3ef9223e 100755 --- a/test/backup_transform.py +++ b/test/backup_transform.py @@ -79,7 +79,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/client_test.sh b/test/client_test.sh index 82be7996c8d..6ad69b2023f 100755 --- a/test/client_test.sh +++ b/test/client_test.sh @@ -17,8 +17,10 @@ # This runs client tests. It used to be part of local_example, # but has been moved to its own test. It hijacks the public examples scripts +source build.env + set -xe -cd "$VTTOP/examples/local" +cd "$VTROOT/examples/local" CELL=test ./etcd-up.sh CELL=test ./vtctld-up.sh diff --git a/test/cluster/k8s_environment.py b/test/cluster/k8s_environment.py index 0951ae58a09..d0026fb70b5 100644 --- a/test/cluster/k8s_environment.py +++ b/test/cluster/k8s_environment.py @@ -121,7 +121,7 @@ def create(self, **kwargs): if 'VITESS_NAME' not in kwargs: kwargs['VITESS_NAME'] = getpass.getuser() kwargs['TEST_MODE'] = '1' - self.script_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes') + self.script_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes') try: subprocess.check_output(['gcloud', 'config', 'list']) except OSError: diff --git a/test/cluster/keytar/Dockerfile b/test/cluster/keytar/Dockerfile deleted file mode 100644 index 06456cd6b7e..00000000000 --- a/test/cluster/keytar/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Dockerfile for generating the keytar image. See README.md for more information. -FROM debian:jessie - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update -y \ - && apt-get install --no-install-recommends -y -q \ - apt-utils \ - apt-transport-https \ - build-essential \ - curl \ - python2.7 \ - python2.7-dev \ - python-pip \ - git \ - wget \ - && pip install -U pip \ - && pip install virtualenv - -RUN echo "deb https://packages.cloud.google.com/apt cloud-sdk-jessie main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -RUN apt-get update -y && apt-get install -y google-cloud-sdk && apt-get install -y kubectl - -WORKDIR /app -RUN virtualenv /env -ADD requirements.txt /app/requirements.txt -RUN /env/bin/pip install -r /app/requirements.txt -ADD keytar.py test_runner.py /app/ -ADD static /app/static - -ENV USER keytar - -ENV PYTHONPATH /env/lib/python2.7/site-packages -ENV CLOUDSDK_PYTHON_SITEPACKAGES $PYTHONPATH - -RUN /bin/bash -c "source ~/.bashrc" - -EXPOSE 8080 -CMD [] -ENTRYPOINT ["/env/bin/python", "keytar.py"] - -ENV PATH /env/bin:$PATH - diff --git a/test/cluster/keytar/README.md b/test/cluster/keytar/README.md deleted file mode 100644 index 081957d192a..00000000000 --- a/test/cluster/keytar/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Keytar - -Keytar is an internally used Vitess system for continuous execution of cluster tests on Kubernetes/Google Cloud. It monitors docker images on [Docker Hub](https://hub.docker.com). When a new image is uploaded to Docker Hub, Keytar starts a cluster on Google Compute Engine (GKE) and runs Kubernetes applications for the purpose of executing cluster tests. It will then locally run tests against the cluster. It exposes a simple web status page showing test results. - -## Setup - -How to set up Keytar for Vitess: - -* Create service account keys with GKE credentials on the account to run the tests on. Follow [step 1 from the GKE developers page](https://developers.google.com/identity/protocols/application-default-credentials?hl=en_US#howtheywork). -* Move the generated keyfile to `$VTTOP/test/cluster/keytar/config`. -* Create or modify the test configuration file (`$VTTOP/test/cluster/keytar/config/vitess_config.yaml`). -* Ensure the configuration has the correct values for GKE project name and keyfile: - ``` - cluster_setup: - - type: gke - project_name: - keyfile: /config/ - ``` -* Then run the following commands: - ``` - > cd $VTTOP/test/cluster/keytar - > KEYTAR_PASSWORD= KEYTAR_PORT= KEYTAR_CONFIG= ./keytar-up.sh - ``` -* Add a Docker Hub webhook pointing to the Keytar service. The webhook URL should be in the form: - ``` - http://:80/test_request?password= - ``` - -## Dashboard - -The script to start Keytar should output a web address to view the current status. If not, the following command can also be run: -```shell -> kubectl get service keytar -o template --template '{{if ge (len .status.loadBalancer) 1}}{{index (index .status.loadBalancer.ingress 0) "ip"}}{{end}}' -``` - -## Limitations - -Currently, Keytar has the following limitations: - -* Only one configuration file allowed at a time. -* Configuration cannot be updated dynamically. -* Test results are saved in memory and are not durable. -* Results are only shown on the dashboard, there is no notification mechanism. diff --git a/test/cluster/keytar/config/vitess_config.yaml b/test/cluster/keytar/config/vitess_config.yaml deleted file mode 100644 index a8b0e8a995b..00000000000 --- a/test/cluster/keytar/config/vitess_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -install: - dependencies: - - python-mysqldb - extra: - - apt-get update - - wget https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz - - tar -C /usr/local -xzf go1.12.7.linux-amd64.tar.gz - - wget https://storage.googleapis.com/kubernetes-helm/helm-v2.1.3-linux-amd64.tar.gz - - tar -zxvf helm-v2.1.3-linux-amd64.tar.gz - - pip install numpy - - pip install selenium - - pip install --upgrade grpcio==1.0.4 - path: - - /usr/local/go/bin - - /app/linux-amd64/ - cluster_setup: - - type: gke - keyfile: /config/keyfile.json -config: - - docker_image: vitess/root - github: - repo: vitessio/vitess - repo_prefix: src/vitess.io/vitess - environment: - sandbox: test/cluster/sandbox/vitess_kubernetes_sandbox.py - config: test/cluster/sandbox/example_sandbox.yaml - cluster_type: gke - application_type: k8s - before_test: - - export VTTOP=$(pwd) - - export VTROOT="${VTROOT:-${VTTOP/\/src\/github.com\/youtube\/vitess/}}" - - export GOPATH=$VTROOT - - export PYTHONPATH=$VTTOP/py:$VTTOP/test:$VTTOP/test/cluster/sandbox:/usr/lib/python2.7/dist-packages:/env/lib/python2.7/site-packages - - go get vitess.io/vitess/go/cmd/vtctlclient - - export PATH=$GOPATH/bin:$PATH - tests: - - file: test/cluster/drain_test.py - params: - num_drains: 1 - - file: test/cluster/backup_test.py - params: - num_backups: 1 - - file: test/cluster/reparent_test.py - params: - num_reparents: 1 diff --git a/test/cluster/keytar/dummy_test.py b/test/cluster/keytar/dummy_test.py deleted file mode 100755 index e0ae6cf6fd9..00000000000 --- a/test/cluster/keytar/dummy_test.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Dummy no-op test to be used in the webdriver test.""" - -import logging -import sys -import unittest - - -class DummyTest(unittest.TestCase): - - def test_dummy(self): - logging.info('Dummy output.') - - -if __name__ == '__main__': - logging.getLogger().setLevel(logging.INFO) - del sys.argv[1:] - unittest.main() diff --git a/test/cluster/keytar/keytar-controller-template.yaml b/test/cluster/keytar/keytar-controller-template.yaml deleted file mode 100644 index ccc6f12e1d7..00000000000 --- a/test/cluster/keytar/keytar-controller-template.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: ReplicationController -apiVersion: v1 -metadata: - name: keytar -spec: - replicas: 1 - template: - metadata: - labels: - component: frontend - app: keytar - spec: - containers: - - name: keytar - image: vitess/keytar - ports: - - name: http-server - containerPort: {{port}} - resources: - limits: - memory: "4Gi" - cpu: "500m" - args: ["--config_file", "{{config}}", "--port", "{{port}}", "--password", "{{password}}"] - volumeMounts: - - name: config - mountPath: /config - volumes: - - name: config - configMap: - name: config diff --git a/test/cluster/keytar/keytar-service.yaml b/test/cluster/keytar/keytar-service.yaml deleted file mode 100644 index 097fafdb947..00000000000 --- a/test/cluster/keytar/keytar-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: keytar - labels: - component: frontend - app: keytar -spec: - ports: - - port: 80 - targetPort: http-server - selector: - component: frontend - app: keytar - type: LoadBalancer diff --git a/test/cluster/keytar/keytar-up.sh b/test/cluster/keytar/keytar-up.sh deleted file mode 100755 index a2f51b05557..00000000000 --- a/test/cluster/keytar/keytar-up.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -KUBECTL=${KUBECTL:-kubectl} - -config_path=${KEYTAR_CONFIG_PATH:-"./config"} -port=${KEYTAR_PORT:-8080} -password=${KEYTAR_PASSWORD:-"defaultkey"} -config=${KEYTAR_CONFIG:-"/config/vitess_config.yaml"} - -sed_script="" -for var in config_path port config password; do - sed_script+="s,{{$var}},${!var},g;" -done - -gcloud container clusters create keytar --machine-type n1-standard-4 --num-nodes 1 --scopes cloud-platform --zone us-central1-b - -echo "Creating keytar configmap" -$KUBECTL create configmap --from-file=$config_path config - -echo "Creating keytar service" -$KUBECTL create -f keytar-service.yaml - -echo "Creating keytar controller" -cat keytar-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - - -echo "Creating firewall-rule" -gcloud compute firewall-rules create keytar --allow tcp:80 - -for i in `seq 1 20`; do - ip=`$KUBECTL get service keytar -o template --template '{{if ge (len .status.loadBalancer) 1}}{{index (index .status.loadBalancer.ingress 0) "ip"}}{{end}}'` - if [[ -n "$ip" ]]; then - echo "Keytar address: http://${ip}:80" - break - fi - echo "Waiting for keytar external IP" - sleep 10 -done diff --git a/test/cluster/keytar/keytar.py b/test/cluster/keytar/keytar.py deleted file mode 100755 index 9b1af17623e..00000000000 --- a/test/cluster/keytar/keytar.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Keytar flask app. - -This program is responsible for exposing an interface to trigger cluster level -tests. For instance, docker webhooks can be configured to point to this -application in order to trigger tests upon pushing new docker images. -""" - -import argparse -import collections -import datetime -import json -import logging -import os -import Queue -import shutil -import subprocess -import tempfile -import threading -import yaml - -import flask - - -app = flask.Flask(__name__) -results = collections.OrderedDict() -_TEMPLATE = ( - 'python {directory}/test_runner.py -c "{config}" -t {timestamp} ' - '-d {tempdir} -s {server}') - - -class KeytarError(Exception): - pass - - -def run_test_config(config): - """Runs a single test iteration from a configuration.""" - tempdir = tempfile.mkdtemp() - logging.info('Fetching github repository') - - # Get the github repo and clone it. - github_config = config['github'] - github_clone_args, github_repo_dir = _get_download_github_repo_args( - tempdir, github_config) - os.makedirs(github_repo_dir) - subprocess.call(github_clone_args) - - current_dir = os.getcwd() - - timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M') - results[timestamp] = { - 'timestamp': timestamp, - 'status': 'Start', - 'tests': {}, - 'docker_image': config['docker_image'] - } - - # Generate a test script with the steps described in the configuration, - # as well as the command to execute the test_runner. - with tempfile.NamedTemporaryFile(dir=tempdir, delete=False) as f: - tempscript = f.name - f.write('#!/bin/bash\n') - if 'before_test' in config: - # Change to the github repo directory, any steps to be run before the - # tests should be executed from there. - os.chdir(github_repo_dir) - for before_step in config['before_test']: - f.write('%s\n' % before_step) - server = 'http://localhost:%d' % app.config['port'] - f.write(_TEMPLATE.format( - directory=current_dir, config=yaml.dump(config), timestamp=timestamp, - tempdir=tempdir, server=server)) - os.chmod(tempscript, 0775) - - try: - subprocess.call([tempscript]) - except subprocess.CalledProcessError as e: - logging.warn('Error running test_runner: %s', str(e)) - finally: - os.chdir(current_dir) - shutil.rmtree(tempdir) - - -@app.route('/') -def index(): - return app.send_static_file('index.html') - - -@app.route('/test_results') -def test_results(): - return json.dumps([results[x] for x in sorted(results)]) - - -@app.route('/test_log') -def test_log(): - # Fetch the output from a test. - log = '%s.log' % os.path.basename(flask.request.values['log_name']) - return (flask.send_from_directory('/tmp/testlogs', log), 200, - {'Content-Type': 'text/css'}) - - -@app.route('/update_results', methods=['POST']) -def update_results(): - # Update the results dict, called from the test_runner. - update_args = flask.request.get_json() - timestamp = update_args['timestamp'] - results[timestamp].update(update_args) - return 'OK' - - -def _validate_request(keytar_password, request_values): - """Checks a request against the password provided to the service at startup. - - Raises an exception on errors, otherwise returns None. - - Args: - keytar_password: password provided to the service at startup. - request_values: dict of POST request values provided to Flask. - - Raises: - KeytarError: raised if the password is invalid. - """ - if keytar_password: - if 'password' not in request_values: - raise KeytarError('Expected password not provided in test_request!') - elif request_values['password'] != keytar_password: - raise KeytarError('Incorrect password passed to test_request!') - - -@app.route('/test_request', methods=['POST']) -def test_request(): - """Respond to a post request to execute tests. - - This expects a json payload containing the docker webhook information. - If this app is configured to use a password, the password should be passed in - as part of the POST request. - - Returns: - HTML response. - """ - try: - _validate_request(app.config['password'], flask.request.values) - except KeytarError as e: - flask.abort(400, str(e)) - webhook_data = flask.request.get_json() - repo_name = webhook_data['repository']['repo_name'] - test_configs = [c for c in app.config['keytar_config']['config'] - if c['docker_image'] == repo_name] - if not test_configs: - return 'No config found for repo_name: %s' % repo_name - for test_config in test_configs: - test_worker.add_test(test_config) - return 'OK' - - -def handle_cluster_setup(cluster_setup): - """Setups up a cluster. - - Currently only GKE is supported. This step handles setting up credentials and - ensuring a valid project name is used. - - Args: - cluster_setup: YAML cluster configuration. - - Raises: - KeytarError: raised on invalid setup configurations. - """ - if cluster_setup['type'] != 'gke': - return - - if 'keyfile' not in cluster_setup: - raise KeytarError('No keyfile found in GKE cluster setup!') - # Add authentication steps to allow keytar to start clusters on GKE. - gcloud_args = ['gcloud', 'auth', 'activate-service-account', - '--key-file', cluster_setup['keyfile']] - logging.info('authenticating using keyfile: %s', cluster_setup['keyfile']) - subprocess.call(gcloud_args) - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = cluster_setup['keyfile'] - - # Ensure that a project name is correctly set. Use the name if provided - # in the configuration, otherwise use the current project name, or else - # the first available project name. - if 'project_name' in cluster_setup: - logging.info('Setting gcloud project to %s', cluster_setup['project_name']) - subprocess.call( - ['gcloud', 'config', 'set', 'project', cluster_setup['project_name']]) - else: - config = subprocess.check_output( - ['gcloud', 'config', 'list', '--format', 'json']) - project_name = json.loads(config)['core']['project'] - if not project_name: - projects = subprocess.check_output(['gcloud', 'projects', 'list']) - first_project = projects[0]['projectId'] - logging.info('gcloud project is unset, setting it to %s', first_project) - subprocess.check_output( - ['gcloud', 'config', 'set', 'project', first_project]) - - -def handle_install_steps(keytar_config): - """Runs all config installation/setup steps. - - Args: - keytar_config: YAML keytar configuration. - """ - if 'install' not in keytar_config: - return - install_config = keytar_config['install'] - for cluster_setup in install_config.get('cluster_setup', []): - handle_cluster_setup(cluster_setup) - - # Install any dependencies using apt-get. - if 'dependencies' in install_config: - subprocess.call(['apt-get', 'update']) - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' - for dep in install_config['dependencies']: - subprocess.call( - ['apt-get', 'install', '-y', '--no-install-recommends', dep]) - - # Run any additional commands if provided. - for step in install_config.get('extra', []): - os.system(step) - - # Update path environment variable. - for path in install_config.get('path', []): - os.environ['PATH'] = '%s:%s' % (path, os.environ['PATH']) - - -def _get_download_github_repo_args(tempdir, github_config): - """Get arguments for github actions. - - Args: - tempdir: Base directory to git clone into. - github_config: Configuration describing the repo, branches, etc. - - Returns: - ([string], string) for arguments to pass to git, and the directory to - clone into. - """ - repo_prefix = github_config.get('repo_prefix', 'github') - repo_dir = os.path.join(tempdir, repo_prefix) - git_args = ['git', 'clone', 'https://github.com/%s' % github_config['repo'], - repo_dir] - if 'branch' in github_config: - git_args += ['-b', github_config['branch']] - return git_args, repo_dir - - -class TestWorker(object): - """A simple test queue. HTTP requests append to this work queue.""" - - def __init__(self): - self.test_queue = Queue.Queue() - self.worker_thread = threading.Thread(target=self.worker_loop) - self.worker_thread.daemon = True - - def worker_loop(self): - # Run forever, executing tests as they are added to the queue. - while True: - item = self.test_queue.get() - run_test_config(item) - self.test_queue.task_done() - - def start(self): - self.worker_thread.start() - - def add_test(self, config): - self.test_queue.put(config) - -test_worker = TestWorker() - - -def main(): - logging.getLogger().setLevel(logging.INFO) - parser = argparse.ArgumentParser(description='Run keytar') - parser.add_argument('--config_file', help='Keytar config file', required=True) - parser.add_argument('--password', help='Password', default=None) - parser.add_argument('--port', help='Port', default=8080, type=int) - keytar_args = parser.parse_args() - with open(keytar_args.config_file, 'r') as yaml_file: - yaml_config = yaml_file.read() - if not yaml_config: - raise ValueError('No valid yaml config!') - keytar_config = yaml.load(yaml_config) - handle_install_steps(keytar_config) - - if not os.path.isdir('/tmp/testlogs'): - os.mkdir('/tmp/testlogs') - - test_worker.start() - - app.config['port'] = keytar_args.port - app.config['password'] = keytar_args.password - app.config['keytar_config'] = keytar_config - - app.run(host='0.0.0.0', port=keytar_args.port, debug=True) - - -if __name__ == '__main__': - main() diff --git a/test/cluster/keytar/keytar_test.py b/test/cluster/keytar/keytar_test.py deleted file mode 100644 index 04769a013b3..00000000000 --- a/test/cluster/keytar/keytar_test.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Keytar tests.""" - -import json -import os -import unittest - -import keytar - - -class KeytarTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.timestamp = '20160101_0000' - if not os.path.isdir('/tmp/testlogs'): - os.mkdir('/tmp/testlogs') - with open( - '/tmp/testlogs/%s_unittest.py.log' % cls.timestamp, 'w') as testlog: - testlog.write('foo') - - def test_validate_request(self): - keytar._validate_request('foo', {'password': 'foo'}) - keytar._validate_request(None, {'password': 'foo'}) - keytar._validate_request(None, {}) - with self.assertRaises(keytar.KeytarError): - keytar._validate_request('foo', {'password': 'foo2'}) - with self.assertRaises(keytar.KeytarError): - keytar._validate_request('foo', {}) - - def test_get_download_github_repo_args(self): - github_config = {'repo': 'vitessio/vitess', 'repo_prefix': 'foo'} - - github_clone_args, repo_dir = ( - keytar._get_download_github_repo_args('/tmp', github_config)) - self.assertEquals( - github_clone_args, - ['git', 'clone', 'https://github.com/vitessio/vitess', '/tmp/foo']) - self.assertEquals('/tmp/foo', repo_dir) - - github_config = { - 'repo': 'vitessio/vitess', 'repo_prefix': 'foo', 'branch': 'bar'} - github_clone_args, repo_dir = ( - keytar._get_download_github_repo_args('/tmp', github_config)) - self.assertEquals( - github_clone_args, - ['git', 'clone', 'https://github.com/vitessio/vitess', '/tmp/foo', '-b', - 'bar']) - self.assertEquals('/tmp/foo', repo_dir) - - def test_logs(self): - # Check GET test_results with no results. - tester = keytar.app.test_client(self) - log = tester.get('/test_log?log_name=%s_unittest.py' % self.timestamp) - self.assertEqual(log.status_code, 200) - self.assertEqual(log.data, 'foo') - - def test_results(self): - # Check GET test_results with no results. - tester = keytar.app.test_client(self) - test_results = tester.get('/test_results') - self.assertEqual(test_results.status_code, 200) - self.assertEqual(json.loads(test_results.data), []) - - # Create a test_result, GET test_results should return an entry now. - keytar.results[self.timestamp] = { - 'timestamp': self.timestamp, - 'status': 'Start', - 'tests': {}, - } - test_results = tester.get('/test_results') - self.assertEqual(test_results.status_code, 200) - self.assertEqual( - json.loads(test_results.data), - [{'timestamp': self.timestamp, 'status': 'Start', 'tests': {}}]) - - # Call POST update_results, GET test_results should return a changed entry. - tester.post( - '/update_results', data=json.dumps(dict( - timestamp='20160101_0000', status='Complete')), - follow_redirects=True, content_type='application/json') - test_results = tester.get('/test_results') - self.assertEqual(test_results.status_code, 200) - self.assertEqual( - json.loads(test_results.data), - [{'timestamp': self.timestamp, 'status': 'Complete', 'tests': {}}]) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/cluster/keytar/keytar_web_test.py b/test/cluster/keytar/keytar_web_test.py deleted file mode 100755 index 5784d3335f5..00000000000 --- a/test/cluster/keytar/keytar_web_test.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A keytar webdriver test.""" - -import json -import logging -import signal -import subprocess -import time -import os -from selenium import webdriver -import unittest -import urllib2 - -import environment - - -class TestKeytarWeb(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.driver = environment.create_webdriver() - port = environment.reserve_ports(1) - keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar') - cls.flask_process = subprocess.Popen( - [os.path.join(keytar_folder, 'keytar.py'), - '--config_file=%s' % os.path.join(keytar_folder, 'test_config.yaml'), - '--port=%d' % port, '--password=foo'], - preexec_fn=os.setsid) - cls.flask_addr = 'http://localhost:%d' % port - - @classmethod - def tearDownClass(cls): - os.killpg(cls.flask_process.pid, signal.SIGTERM) - cls.driver.quit() - - def _wait_for_complete_status(self, timeout_s=180): - start_time = time.time() - while time.time() - start_time < timeout_s: - if 'Complete' in self.driver.find_element_by_id('results').text: - return - self.driver.refresh() - time.sleep(5) - self.fail('Timed out waiting for test to finish.') - - def test_keytar_web(self): - self.driver.get(self.flask_addr) - req = urllib2.Request('%s/test_request?password=foo' % self.flask_addr) - req.add_header('Content-Type', 'application/json') - urllib2.urlopen( - req, json.dumps({'repository': {'repo_name': 'test/image'}})) - self._wait_for_complete_status() - logging.info('Dummy test complete.') - self.driver.find_element_by_partial_link_text('PASSED').click() - self.assertIn('Dummy output.', - self.driver.find_element_by_tag_name('body').text) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/cluster/keytar/requirements.txt b/test/cluster/keytar/requirements.txt deleted file mode 100644 index 310546af9c3..00000000000 --- a/test/cluster/keytar/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Flask==1.0 -pyyaml==4.2b1 diff --git a/test/cluster/keytar/static/index.html b/test/cluster/keytar/static/index.html deleted file mode 100644 index 153b752ae55..00000000000 --- a/test/cluster/keytar/static/index.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - Keytar - - - - -
-

Waiting for test results...

-
- - - - - diff --git a/test/cluster/keytar/static/script.js b/test/cluster/keytar/static/script.js deleted file mode 100644 index 29275ffd418..00000000000 --- a/test/cluster/keytar/static/script.js +++ /dev/null @@ -1,42 +0,0 @@ -$(document).ready(function() { - var resultsElement = $("#test-results"); - - var appendTestResults = function(data) { - resultsElement.empty(); - var html = " \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - "; - $.each(data, function(key, value) { - html += ""; - }); - html += "
TimeDocker ImageSandbox NameStatusTestsResults
" + value.timestamp + "" + value.docker_image + "" + value.name + "" + value.status + ""; - $.each(value.tests, function(key, val) { - html += ""; - }); - html += "
" + key + "
"; - $.each(value.tests, function(key, val) { - html += ""; - }); - html += "
" + val + "
"; - resultsElement.append(html); - }; - - // Poll every second. - var fetchTestResults = function() { - $.getJSON("/test_results").done(appendTestResults).always( - function() { - setTimeout(fetchTestResults, 60000); - }); - }; - fetchTestResults(); -}); diff --git a/test/cluster/keytar/static/style.css b/test/cluster/keytar/static/style.css deleted file mode 100644 index fd1c393fb08..00000000000 --- a/test/cluster/keytar/static/style.css +++ /dev/null @@ -1,61 +0,0 @@ -body, input { - color: #123; - font-family: "Gill Sans", sans-serif; -} - -div { - overflow: hidden; - padding: 1em 0; - position: relative; - text-align: center; -} - -h1, h2, p, input, a { - font-weight: 300; - margin: 0; -} - -h1 { - color: #BDB76B; - font-size: 3.5em; -} - -h2 { - color: #999; -} - -form { - margin: 0 auto; - max-width: 50em; - text-align: center; -} - -input { - border: 0; - border-radius: 1000px; - box-shadow: inset 0 0 0 2px #BDB76B; - display: inline; - font-size: 1.5em; - margin-bottom: 1em; - outline: none; - padding: .5em 5%; - width: 55%; -} - -form a { - background: #BDB76B; - border: 0; - border-radius: 1000px; - color: #FFF; - font-size: 1.25em; - font-weight: 400; - padding: .75em 2em; - text-decoration: none; - text-transform: uppercase; - white-space: normal; -} - -p { - font-size: 1.5em; - line-height: 1.5; -} diff --git a/test/cluster/keytar/test_config.yaml b/test/cluster/keytar/test_config.yaml deleted file mode 100644 index 308c6de7026..00000000000 --- a/test/cluster/keytar/test_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -install: - path: - - /test_path -config: - - docker_image: test/image - github: - repo: vitessio/vitess - repo_prefix: src/vitess.io/vitess - before_test: - - touch /tmp/test_file - environment: - cluster_type: gke - application_type: k8s - tests: - - file: test/cluster/keytar/dummy_test.py diff --git a/test/cluster/keytar/test_runner.py b/test/cluster/keytar/test_runner.py deleted file mode 100755 index 79d8d496e04..00000000000 --- a/test/cluster/keytar/test_runner.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Script to run a single cluster test. - -This includes the following steps: - 1. Starting a test cluster (GKE supported). - 2. Running tests against the cluster. - 3. Reporting test results. -""" - -import argparse -import json -import logging -import os -import subprocess -import urllib2 -import uuid -import yaml - -keytar_args = None - - -def update_result(k, v): - """Post a key/value pair test result update.""" - url = '%s/update_results' % keytar_args.server - req = urllib2.Request(url) - req.add_header('Content-Type', 'application/json') - urllib2.urlopen(req, json.dumps({k: v, 'timestamp': keytar_args.timestamp})) - - -def run_sandbox_action(environment_config, name, action): - """Run a sandbox action (Start/Stop). - - Args: - environment_config: yaml configuration for the sandbox. - name: unique name for the sandbox. - action: action to pass to the sandbox action parameter. - """ - if 'sandbox' not in environment_config: - return - # Execute sandbox command - sandbox_file = os.path.join(repo_dir, environment_config['sandbox']) - os.chdir(os.path.dirname(sandbox_file)) - sandbox_args = [ - './%s' % os.path.basename(sandbox_file), - '-e', environment_config['cluster_type'], '-n', name, '-k', name, - '-c', os.path.join(repo_dir, environment_config['config']), - '-a', action] - update_result('status', 'Running sandbox action: %s' % action) - try: - subprocess.check_call(sandbox_args) - update_result('status', 'Finished sandbox action: %s' % action) - except subprocess.CalledProcessError as e: - logging.info('Failed to run sandbox action %s: %s', (action, e.output)) - update_result('status', 'Sandbox failure') - - -def run_test_config(): - """Runs a single test iteration from a configuration. - - This includes bringing up an environment, running the tests, and reporting - status. - """ - # Generate a random name. Kubernetes/GKE has name length limits. - name = 'keytar%s' % format(uuid.uuid4().fields[0], 'x') - update_result('name', name) - - environment_config = config['environment'] - run_sandbox_action(environment_config, name, 'Start') - logging.info('Running tests') - update_result('status', 'Running Tests') - - try: - # Run tests and update results. - test_results = {} - for test in config['tests']: - test_file = os.path.join(repo_dir, test['file']) - test_name = os.path.basename(test_file) - logging.info('Running test %s', test_name) - os.chdir(os.path.dirname(test_file)) - test_args = [ - './%s' % test_name, - '-e', environment_config['application_type'], '-n', name] - if 'params' in test: - test_args += ['-t', ':'.join( - '%s=%s' % (k, v) for (k, v) in test['params'].iteritems())] - testlog = '/tmp/testlogs/%s_%s.log' % (keytar_args.timestamp, test_name) - logging.info('Saving log to %s', testlog) - test_results[test_name] = 'RUNNING' - update_result('tests', test_results) - with open(testlog, 'w') as results_file: - if subprocess.call(test_args, stdout=results_file, stderr=results_file): - test_results[test_name] = 'FAILED' - else: - test_results[test_name] = 'PASSED' - update_result('tests', test_results) - update_result('status', 'Tests Complete') - except Exception as e: # pylint: disable=broad-except - logging.info('Exception caught: %s', str(e)) - update_result('status', 'System Error running tests: %s' % str(e)) - finally: - run_sandbox_action(environment_config, name, 'Stop') - - -if __name__ == '__main__': - logging.getLogger().setLevel(logging.INFO) - parser = argparse.ArgumentParser(description='Run keytar') - parser.add_argument('-c', '--config', help='Keytar config yaml') - parser.add_argument('-t', '--timestamp', help='Timestamp string') - parser.add_argument('-d', '--dir', help='temp dir created for the test') - parser.add_argument('-s', '--server', help='keytar server address') - keytar_args = parser.parse_args() - config = yaml.load(keytar_args.config) - repo_prefix = config['github'].get('repo_prefix', 'github') - repo_dir = os.path.join(keytar_args.dir, repo_prefix) - - run_test_config() diff --git a/test/cluster/sandbox/create_schema.py b/test/cluster/sandbox/create_schema.py index 198858eeffe..d60839bd013 100755 --- a/test/cluster/sandbox/create_schema.py +++ b/test/cluster/sandbox/create_schema.py @@ -33,7 +33,7 @@ def main(): parser.add_option( '-s', '--sql_file', help='File containing sql schema', default=os.path.join( - os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql')) + os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql')) logging.getLogger().setLevel(logging.INFO) options, _ = parser.parse_args() diff --git a/test/cluster/sandbox/kubernetes_components.py b/test/cluster/sandbox/kubernetes_components.py index e67c4bf379c..94d3ac5e947 100755 --- a/test/cluster/sandbox/kubernetes_components.py +++ b/test/cluster/sandbox/kubernetes_components.py @@ -81,7 +81,7 @@ def start(self): logging.info('Installing helm.') try: subprocess.check_output( - ['helm', 'install', os.path.join(os.environ['VTTOP'], 'helm/vitess'), + ['helm', 'install', os.path.join(os.environ['VTROOT'], 'helm/vitess'), '-n', self.sandbox_name, '--namespace', self.sandbox_name, '--replace', '--values', self.helm_config], stderr=subprocess.STDOUT) diff --git a/test/cluster/sandbox/vitess_kubernetes_sandbox.py b/test/cluster/sandbox/vitess_kubernetes_sandbox.py index 734f8d1ac46..016a8d99916 100755 --- a/test/cluster/sandbox/vitess_kubernetes_sandbox.py +++ b/test/cluster/sandbox/vitess_kubernetes_sandbox.py @@ -46,7 +46,7 @@ def generate_guestbook_sandlet(self): """Creates a sandlet encompassing the guestbook app built on Vitess.""" guestbook_sandlet = sandlet.Sandlet('guestbook') guestbook_sandlet.dependencies = ['helm'] - template_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes') + template_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes') guestbook_sandlet.components.add_component( self.cluster_env.Port('%s-guestbook' % self.name, 80)) for keyspace in self.app_options.keyspaces: @@ -54,7 +54,7 @@ def generate_guestbook_sandlet(self): 'create_schema_%s' % keyspace['name'], self.name, 'create_schema.py', self.log_dir, namespace=self.name, keyspace=keyspace['name'], drop_table='messages', sql_file=os.path.join( - os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql')) + os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql')) guestbook_sandlet.components.add_component(create_schema_subprocess) guestbook_sandlet.components.add_component( kubernetes_components.KubernetesResource( diff --git a/test/cluster/sandbox/vtctl_sandbox.py b/test/cluster/sandbox/vtctl_sandbox.py index ef0f1d978ed..3e495b414ab 100755 --- a/test/cluster/sandbox/vtctl_sandbox.py +++ b/test/cluster/sandbox/vtctl_sandbox.py @@ -68,7 +68,7 @@ def execute_vtctl_command(vtctl_args, namespace='default', timeout_s=180): # Default to trying to use kvtctl.sh if a forwarded port cannot be found. os.environ['VITESS_NAME'] = namespace vtctl_cmd_args = ( - [os.path.join(os.environ['VTTOP'], 'examples/kubernetes/kvtctl.sh')] + [os.path.join(os.environ['VTROOT'], 'examples/kubernetes/kvtctl.sh')] + vtctl_args) start_time = time.time() diff --git a/test/config.json b/test/config.json index 0974ba92c3f..62a02765f8f 100644 --- a/test/config.json +++ b/test/config.json @@ -94,21 +94,10 @@ "tools/check_make_parser.sh" ], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 1, "Tags": [] }, - "custom_sharding": { - "File": "custom_sharding.py", - "Args": [], - "Command": [], - "Manual": false, - "Shard": 4, - "RetryMax": 0, - "Tags": [ - "worker_test" - ] - }, "encrypted_replication": { "File": "encrypted_replication.py", "Args": [], @@ -188,7 +177,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [ "site_test" @@ -212,7 +201,7 @@ "test/local_example.sh" ], "Manual": false, - "Shard": 3, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -294,7 +283,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -303,7 +292,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [ "site_test" @@ -383,7 +372,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -401,7 +390,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 2, + "Shard": 5, "RetryMax": 0, "Tags": [ "site_test" @@ -440,7 +429,19 @@ "tools/e2e_test_runner.sh" ], "Manual": false, - "Shard": 3, + "Shard": 5, + "RetryMax": 0, + "Tags": [] + }, + "cluster_endtoend": { + "File": "", + "Args": [], + "Command": [ + "make", + "e2e_test_cluster" + ], + "Manual": false, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -452,7 +453,7 @@ "e2e_test_race" ], "Manual": false, - "Shard": 1, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -463,7 +464,7 @@ "tools/unit_test_runner.sh" ], "Manual": false, - "Shard": 0, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -475,7 +476,7 @@ "unit_test_race" ], "Manual": false, - "Shard": 3, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -582,7 +583,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [] }, @@ -600,7 +601,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 2, + "Shard": 5, "RetryMax": 0, "Tags": [ "worker_test" diff --git a/test/custom_sharding.py b/test/custom_sharding.py deleted file mode 100755 index a3197cbb2e5..00000000000 --- a/test/custom_sharding.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import unittest - -import environment -import tablet -import utils - -from vtproto import topodata_pb2 - -from vtdb import vtgate_client - -# shards need at least 1 replica for semi-sync ACK, and 1 rdonly for SplitQuery. -shard_0_master = tablet.Tablet() -shard_0_replica = tablet.Tablet() -shard_0_rdonly = tablet.Tablet() - -shard_1_master = tablet.Tablet() -shard_1_replica = tablet.Tablet() -shard_1_rdonly = tablet.Tablet() - -all_tablets = [shard_0_master, shard_0_replica, shard_0_rdonly, - shard_1_master, shard_1_replica, shard_1_rdonly] - - -def setUpModule(): - try: - environment.topo_server().setup() - - setup_procs = [t.init_mysql() for t in all_tablets] - utils.Vtctld().start() - utils.wait_procs(setup_procs) - except: - tearDownModule() - raise - - -def tearDownModule(): - utils.required_teardown() - if utils.options.skip_teardown: - return - - teardown_procs = [t.teardown_mysql() for t in all_tablets] - utils.wait_procs(teardown_procs, raise_on_error=False) - - environment.topo_server().teardown() - utils.kill_sub_processes() - utils.remove_tmp_files() - - for t in all_tablets: - t.remove_tree() - - -class TestCustomSharding(unittest.TestCase): - """Test a custom-shared keyspace.""" - - def _vtdb_conn(self): - protocol, addr = utils.vtgate.rpc_endpoint(python=True) - return vtgate_client.connect(protocol, addr, 30.0) - - def _insert_data(self, shard, start, count, table='data'): - sql = 'insert into ' + table + '(id, name) values (:id, :name)' - conn = self._vtdb_conn() - cursor = conn.cursor( - tablet_type='master', keyspace='test_keyspace', - shards=[shard], - writable=True) - for x in xrange(count): - bindvars = { - 'id': start+x, - 'name': 'row %d' % (start+x), - } - conn.begin() - cursor.execute(sql, bindvars) - conn.commit() - conn.close() - - def _check_data(self, shard, start, count, table='data'): - sql = 'select name from ' + table + ' where id=:id' - conn = self._vtdb_conn() - cursor = conn.cursor( - tablet_type='master', keyspace='test_keyspace', - shards=[shard]) - for x in xrange(count): - bindvars = { - 'id': start+x, - } - cursor.execute(sql, bindvars) - qr = cursor.fetchall() - self.assertEqual(len(qr), 1) - v = qr[0][0] - self.assertEqual(v, 'row %d' % (start+x)) - conn.close() - - def test_custom_end_to_end(self): - """Runs through the common operations of a custom sharded keyspace. - - Tests creation with one shard, schema change, reading / writing - data, adding one more shard, reading / writing data from both - shards, applying schema changes again, and reading / writing data - from both shards again. - """ - - utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) - - # start the first shard only for now - shard_0_master.init_tablet( - 'replica', - keyspace='test_keyspace', - shard='0', - tablet_index=0) - shard_0_replica.init_tablet( - 'replica', - keyspace='test_keyspace', - shard='0', - tablet_index=1) - shard_0_rdonly.init_tablet( - 'rdonly', - keyspace='test_keyspace', - shard='0', - tablet_index=2) - - for t in [shard_0_master, shard_0_replica, shard_0_rdonly]: - t.create_db('vt_test_keyspace') - t.start_vttablet(wait_for_state=None) - - for t in [shard_0_master, shard_0_replica, shard_0_rdonly]: - t.wait_for_vttablet_state('NOT_SERVING') - - utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/0', - shard_0_master.tablet_alias], auto_log=True) - utils.wait_for_tablet_type(shard_0_replica.tablet_alias, 'replica') - utils.wait_for_tablet_type(shard_0_rdonly.tablet_alias, 'rdonly') - for t in [shard_0_master, shard_0_replica, shard_0_rdonly]: - t.wait_for_vttablet_state('SERVING') - - self._check_shards_count_in_srv_keyspace(1) - s = utils.run_vtctl_json(['GetShard', 'test_keyspace/0']) - self.assertEqual(s['is_master_serving'], True) - - # create a table on shard 0 - sql = '''create table data( -id bigint auto_increment, -name varchar(64), -primary key (id) -) Engine=InnoDB''' - utils.run_vtctl(['ApplySchema', '-sql=' + sql, 'test_keyspace'], - auto_log=True) - - # reload schema everywhere so the QueryService knows about the tables - for t in [shard_0_master, shard_0_replica, shard_0_rdonly]: - utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True) - - # create shard 1 - shard_1_master.init_tablet( - 'replica', - keyspace='test_keyspace', - shard='1', - tablet_index=0) - shard_1_replica.init_tablet( - 'replica', - keyspace='test_keyspace', - shard='1', - tablet_index=1) - shard_1_rdonly.init_tablet( - 'rdonly', - keyspace='test_keyspace', - shard='1', - tablet_index=2) - - for t in [shard_1_master, shard_1_replica, shard_1_rdonly]: - t.create_db('vt_test_keyspace') - t.start_vttablet(wait_for_state=None) - - for t in [shard_1_master, shard_1_replica, shard_1_rdonly]: - t.wait_for_vttablet_state('NOT_SERVING') - - s = utils.run_vtctl_json(['GetShard', 'test_keyspace/1']) - self.assertEqual(s['is_master_serving'], True) - - utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/1', - shard_1_master.tablet_alias], auto_log=True) - utils.wait_for_tablet_type(shard_1_replica.tablet_alias, 'replica') - utils.wait_for_tablet_type(shard_1_rdonly.tablet_alias, 'rdonly') - for t in [shard_1_master, shard_1_replica, shard_1_rdonly]: - t.wait_for_vttablet_state('SERVING') - utils.run_vtctl(['CopySchemaShard', shard_0_rdonly.tablet_alias, - 'test_keyspace/1'], auto_log=True) - - # we need to rebuild SrvKeyspace here to account for the new shards. - utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) - self._check_shards_count_in_srv_keyspace(2) - - # must start vtgate after tablets are up, or else wait until 1min refresh - utils.VtGate().start(tablets=[ - shard_0_master, shard_0_replica, shard_0_rdonly, - shard_1_master, shard_1_replica, shard_1_rdonly]) - utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1) - utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1) - utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1) - utils.vtgate.wait_for_endpoints('test_keyspace.1.master', 1) - utils.vtgate.wait_for_endpoints('test_keyspace.1.replica', 1) - utils.vtgate.wait_for_endpoints('test_keyspace.1.rdonly', 1) - - # insert and check data on shard 0 - self._insert_data('0', 100, 10) - self._check_data('0', 100, 10) - - # insert and check data on shard 1 - self._insert_data('1', 200, 10) - self._check_data('1', 200, 10) - - # create a second table on all shards - sql = '''create table data2( -id bigint auto_increment, -name varchar(64), -primary key (id) -) Engine=InnoDB''' - utils.run_vtctl(['ApplySchema', '-sql=' + sql, 'test_keyspace'], - auto_log=True) - - # reload schema everywhere so the QueryService knows about the tables - for t in all_tablets: - utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True) - - # insert and read data on all shards - self._insert_data('0', 300, 10, table='data2') - self._insert_data('1', 400, 10, table='data2') - self._check_data('0', 300, 10, table='data2') - self._check_data('1', 400, 10, table='data2') - - # Now test SplitQuery API works (used in MapReduce usually, but bringing - # up a full MR-capable cluster is too much for this test environment) - sql = 'select id, name from data' - s = utils.vtgate.split_query(sql, 'test_keyspace', 4) - self.assertEqual(len(s), 4) - shard0count = 0 - shard1count = 0 - for q in s: - if q['shard_part']['shards'][0] == '0': - shard0count += 1 - if q['shard_part']['shards'][0] == '1': - shard1count += 1 - self.assertEqual(shard0count, 2) - self.assertEqual(shard1count, 2) - - # run the queries, aggregate the results, make sure we have all rows - rows = {} - for q in s: - bindvars = {} - for name, value in q['query']['bind_variables'].iteritems(): - # vtctl encodes bytes as base64. - bindvars[name] = int(base64.standard_b64decode(value['value'])) - qr = utils.vtgate.execute_shards( - q['query']['sql'], - 'test_keyspace', ','.join(q['shard_part']['shards']), - tablet_type='master', bindvars=bindvars) - for r in qr['rows']: - rows[int(r[0])] = r[1] - self.assertEqual(len(rows), 20) - expected = {} - for i in xrange(10): - expected[100 + i] = 'row %d' % (100 + i) - expected[200 + i] = 'row %d' % (200 + i) - self.assertEqual(rows, expected) - - def _check_shards_count_in_srv_keyspace(self, shard_count): - ks = utils.run_vtctl_json(['GetSrvKeyspace', 'test_nj', 'test_keyspace']) - check_types = set([topodata_pb2.MASTER, topodata_pb2.REPLICA, - topodata_pb2.RDONLY]) - for p in ks['partitions']: - if p['served_type'] in check_types: - self.assertEqual(len(p['shard_references']), shard_count) - check_types.remove(p['served_type']) - - self.assertEqual(len(check_types), 0, - 'The number of expected shard_references in GetSrvKeyspace' - ' was not equal %d for all expected tablet types.' - % shard_count) - - -if __name__ == '__main__': - utils.main() diff --git a/test/environment.py b/test/environment.py index ce3b58db683..eee51cc4523 100644 --- a/test/environment.py +++ b/test/environment.py @@ -53,14 +53,6 @@ 'ERROR: Vitess and mysqld ' 'should not be run as root.\n') sys.exit(1) -if 'VTTOP' not in os.environ: - sys.stderr.write( - 'ERROR: Vitess environment not set up. ' - 'Please run "source dev.env" first.\n') - sys.exit(1) - -# vttop is the toplevel of the vitess source tree -vttop = os.environ['VTTOP'] # vtroot is where everything gets installed vtroot = os.environ['VTROOT'] @@ -162,7 +154,7 @@ def prog_compile(name): return compiled_progs.append(name) logging.debug('Compiling %s', name) - run(['go', 'install'], cwd=os.path.join(vttop, 'go', 'cmd', name)) + run(['go', 'install'], cwd=os.path.join(vtroot, 'go', 'cmd', name)) # binary management: returns the full path for a binary this should diff --git a/test/initial_sharding_multi.py b/test/initial_sharding_multi.py index af0b167ed2d..8edad5a73e5 100755 --- a/test/initial_sharding_multi.py +++ b/test/initial_sharding_multi.py @@ -127,7 +127,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/keyspace_test.py b/test/keyspace_test.py index bc35027adc2..4fedac8f4ef 100755 --- a/test/keyspace_test.py +++ b/test/keyspace_test.py @@ -301,23 +301,15 @@ def test_remove_keyspace_cell(self): utils.run_vtctl( ['InitTablet', '-port=1234', '-keyspace=test_delete_keyspace', '-shard=0', 'test_ca-0000000100', 'master']) - utils.run_vtctl( - ['InitTablet', '-port=1234', '-keyspace=test_delete_keyspace', - '-shard=1', 'test_ca-0000000101', 'master']) utils.run_vtctl( ['InitTablet', '-port=1234', '-keyspace=test_delete_keyspace', '-shard=0', 'test_nj-0000000100', 'replica']) - utils.run_vtctl( - ['InitTablet', '-port=1234', '-keyspace=test_delete_keyspace', - '-shard=1', 'test_nj-0000000101', 'replica']) # Create the serving/replication entries and check that they exist, # so we can later check they're deleted. utils.run_vtctl(['RebuildKeyspaceGraph', 'test_delete_keyspace']) utils.run_vtctl( ['GetShardReplication', 'test_nj', 'test_delete_keyspace/0']) - utils.run_vtctl( - ['GetShardReplication', 'test_nj', 'test_delete_keyspace/1']) utils.run_vtctl(['GetSrvKeyspace', 'test_nj', 'test_delete_keyspace']) utils.run_vtctl(['GetSrvKeyspace', 'test_ca', 'test_delete_keyspace']) @@ -328,13 +320,13 @@ def test_remove_keyspace_cell(self): # Check that the shard is gone from test_nj. srv_keyspace = utils.run_vtctl_json(['GetSrvKeyspace', 'test_nj', 'test_delete_keyspace']) for partition in srv_keyspace['partitions']: - self.assertEqual(len(partition['shard_references']), 1, + self.assertEqual(len(partition['shard_references']), 0, 'RemoveShardCell should have removed one shard from the target cell: ' + json.dumps(srv_keyspace)) # Make sure the shard is still serving in test_ca. srv_keyspace = utils.run_vtctl_json(['GetSrvKeyspace', 'test_ca', 'test_delete_keyspace']) for partition in srv_keyspace['partitions']: - self.assertEqual(len(partition['shard_references']), 2, + self.assertEqual(len(partition['shard_references']), 1, 'RemoveShardCell should not have changed other cells: ' + json.dumps(srv_keyspace)) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_delete_keyspace']) @@ -343,14 +335,11 @@ def test_remove_keyspace_cell(self): utils.run_vtctl(['GetShard', 'test_delete_keyspace/0']) utils.run_vtctl(['GetTablet', 'test_ca-0000000100']) utils.run_vtctl(['GetTablet', 'test_nj-0000000100'], expect_fail=True) - utils.run_vtctl(['GetTablet', 'test_nj-0000000101']) utils.run_vtctl( ['GetShardReplication', 'test_ca', 'test_delete_keyspace/0']) utils.run_vtctl( ['GetShardReplication', 'test_nj', 'test_delete_keyspace/0'], expect_fail=True) - utils.run_vtctl( - ['GetShardReplication', 'test_nj', 'test_delete_keyspace/1']) utils.run_vtctl(['GetSrvKeyspace', 'test_nj', 'test_delete_keyspace']) # Add it back to do another test. @@ -372,9 +361,6 @@ def test_remove_keyspace_cell(self): utils.run_vtctl( ['GetShardReplication', 'test_nj', 'test_delete_keyspace/0'], expect_fail=True) - utils.run_vtctl( - ['GetShardReplication', 'test_nj', 'test_delete_keyspace/1'], - expect_fail=True) # Clean up. utils.run_vtctl(['DeleteKeyspace', '-recursive', 'test_delete_keyspace']) diff --git a/test/local_example.sh b/test/local_example.sh index f0ba0f4278b..26c41abf15f 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -18,9 +18,11 @@ # It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/ # So we can detect if a regression affecting a tutorial is introduced. +source build.env + set -xe -cd "$VTTOP/examples/local" +cd "$VTROOT/examples/local" ./101_initial_cluster.sh diff --git a/test/mysql_flavor.py b/test/mysql_flavor.py index 9cf057982ea..8102ad0cfd1 100644 --- a/test/mysql_flavor.py +++ b/test/mysql_flavor.py @@ -126,7 +126,7 @@ def reset_replication_commands(self): ] def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mariadb100.cnf" + return "" def master_position(self, tablet): gtid = tablet.mquery("", "SELECT @@GLOBAL.gtid_binlog_pos")[0][0] @@ -152,7 +152,7 @@ class MariaDB103(MariaDB): """Overrides specific to MariaDB 10.3+.""" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mariadb103.cnf" + return "" class MySQL56(MysqlFlavor): """Overrides specific to MySQL 5.6/5.7""" @@ -172,7 +172,7 @@ def position_at_least(self, a, b): ]).strip() == "true" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mysql56.cnf" + return "" def change_master_commands(self, host, port, pos): gtid = pos.split("/")[1] @@ -186,7 +186,7 @@ def change_master_commands(self, host, port, pos): class MySQL80(MySQL56): """Overrides specific to MySQL 8.0.""" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mysql80.cnf" + return "" def change_passwords(self, password_col): """set real passwords for all users""" return ''' diff --git a/test/tablet.py b/test/tablet.py index 552c7d5031d..ba7ff2716ee 100644 --- a/test/tablet.py +++ b/test/tablet.py @@ -55,7 +55,7 @@ def get_backup_storage_flags(): def get_all_extra_my_cnf(extra_my_cnf): - all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf'] + all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf'] flavor_my_cnf = mysql_flavor().extra_my_cnf() if flavor_my_cnf: all_extra_my_cnf.append(flavor_my_cnf) @@ -186,12 +186,12 @@ def init_mysql(self, extra_my_cnf=None, init_db=None, extra_args=None, """ if use_rbr: if extra_my_cnf: - extra_my_cnf += ':' + environment.vttop + '/config/mycnf/rbr.cnf' + extra_my_cnf += ':' + environment.vtroot + '/config/mycnf/rbr.cnf' else: - extra_my_cnf = environment.vttop + '/config/mycnf/rbr.cnf' + extra_my_cnf = environment.vtroot + '/config/mycnf/rbr.cnf' if not init_db: - init_db = environment.vttop + '/config/init_db.sql' + init_db = environment.vtroot + '/config/init_db.sql' if self.use_mysqlctld: self.mysqlctld_process = self.mysqlctld(['-init_db_sql_file', init_db], diff --git a/test/utils.py b/test/utils.py index 0a162375fff..8f645c6cbd8 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1214,8 +1214,8 @@ def start(self, enable_schema_change_dir=False, extra_flags=None): args = environment.binary_args('vtctld') + [ '-enable_queries', '-cell', 'test_nj', - '-web_dir', environment.vttop + '/web/vtctld', - '-web_dir2', environment.vttop + '/web/vtctld2', + '-web_dir', environment.vtroot + '/web/vtctld', + '-web_dir2', environment.vtroot + '/web/vtctld2', '--log_dir', environment.vtlogroot, '--port', str(self.port), '-tablet_manager_protocol', diff --git a/test/vtbackup.py b/test/vtbackup.py index f42831ea4e0..fecaca6612d 100644 --- a/test/vtbackup.py +++ b/test/vtbackup.py @@ -35,7 +35,7 @@ def get_backup_storage_flags(): os.path.join(environment.tmproot, 'backupstorage')] def get_all_extra_my_cnf(extra_my_cnf): - all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf'] + all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf'] flavor_my_cnf = mysql_flavor().extra_my_cnf() if flavor_my_cnf: all_extra_my_cnf.append(flavor_my_cnf) diff --git a/test/vtctld_web_test.py b/test/vtctld_web_test.py index 19738abf77f..33fd8e00861 100755 --- a/test/vtctld_web_test.py +++ b/test/vtctld_web_test.py @@ -86,12 +86,12 @@ def setUpClass(cls): cls.db = local_database.LocalDatabase( topology, - os.path.join(environment.vttop, 'test/vttest_schema'), + os.path.join(environment.vtroot, 'test/vttest_schema'), False, None, - web_dir=os.path.join(environment.vttop, 'web/vtctld'), + web_dir=os.path.join(environment.vtroot, 'web/vtctld'), default_schema_dir=os.path.join( - environment.vttop, 'test/vttest_schema/default'), - web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app')) + environment.vtroot, 'test/vttest_schema/default'), + web_dir2=os.path.join(environment.vtroot, 'web/vtctld2/app')) cls.db.setup() cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port'] utils.pause('Paused test after vtcombo was started.\n' diff --git a/test/vttest_sample_test.py b/test/vttest_sample_test.py index 98888a97777..a4a7de4639f 100755 --- a/test/vttest_sample_test.py +++ b/test/vttest_sample_test.py @@ -82,9 +82,9 @@ def test_standalone(self): '--port', str(port), '--proto_topo', text_format.MessageToString(topology, as_one_line=True), - '--schema_dir', os.path.join(environment.vttop, 'test', + '--schema_dir', os.path.join(environment.vtroot, 'test', 'vttest_schema'), - '--web_dir', environment.vttop + '/web/vtctld', + '--web_dir', environment.vtroot + '/web/vtctld', ] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) config = json.loads(sp.stdout.readline()) diff --git a/tools/bootstrap_web.sh b/tools/bootstrap_web.sh index aecc89b8f80..cc0e40966c9 100755 --- a/tools/bootstrap_web.sh +++ b/tools/bootstrap_web.sh @@ -39,12 +39,12 @@ else # Add the node directory to PATH to make sure that the Angular # installation below can find the "node" binary. # (dev.env does actually append it to PATH.) - source $VTTOP/dev.env + source $VTROOT/dev.env fi echo "Installing dependencies for building web UI" angular_cli_dir=$VTROOT/dist/angular-cli -web_dir2=$VTTOP/web/vtctld2 +web_dir2=$VTROOT/web/vtctld2 angular_cli_commit=cacaa4eff10e135016ef81076fab1086a3bce92f if [[ -d $angular_cli_dir && `cd $angular_cli_dir && git rev-parse HEAD` == "$angular_cli_commit" ]]; then echo "skipping angular cli download. remove $angular_cli_dir to force download." diff --git a/tools/check_make_parser.sh b/tools/check_make_parser.sh index d28d4f18f09..ef59eb67a9f 100755 --- a/tools/check_make_parser.sh +++ b/tools/check_make_parser.sh @@ -6,6 +6,8 @@ # This is used in Travis to verify that the currently committed version was # generated with the proper version of goyacc. +source build.env + CUR="sql.go" TMP="/tmp/sql.$$.go" diff --git a/test/cluster/keytar/keytar-down.sh b/tools/dependency_check.sh similarity index 56% rename from test/cluster/keytar/keytar-down.sh rename to tools/dependency_check.sh index d82a4b08f3d..7d5179c1616 100755 --- a/test/cluster/keytar/keytar-down.sh +++ b/tools/dependency_check.sh @@ -14,10 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -KUBECTL=${KUBECTL:-kubectl} +source build.env -$KUBECTL delete replicationcontroller keytar -$KUBECTL delete service keytar -$KUBECTL delete configmap config -gcloud container clusters delete keytar -z us-central1-b -q -gcloud compute firewall-rules delete keytar -q +function fail() { + echo "ERROR: $1" + exit 1 +} + +# These binaries are required to 'make test' +# mysqld might be in /usr/sbin which will not be in the default PATH +PATH="/usr/sbin:$PATH" +for binary in mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do + command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/contributing/build-from-source for install instructions." +done; diff --git a/tools/e2e_test_cluster.sh b/tools/e2e_test_cluster.sh new file mode 100755 index 00000000000..991cd29ce53 --- /dev/null +++ b/tools/e2e_test_cluster.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These test uses excutables and launch them as process +# After that all tests run, here we are testing those + +# All Go packages with test files. +# Output per line: * + +source build.env + +packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort) + +cluster_tests=$(echo "$packages_with_tests" | grep -E "go/test/endtoend" | cut -d" " -f1) + +# Run cluster test sequentially +echo "running cluster tests $cluster_tests" +echo "$cluster_tests" | xargs go test -v -p=1 +if [ $? -ne 0 ]; then + echo "ERROR: Go cluster tests failed. See above for errors." + echo + echo "This should NOT happen. Did you introduce a flaky unit test?" + echo "If so, please rename it to the suffix _flaky_test.go." + exit 1 +fi diff --git a/tools/e2e_test_race.sh b/tools/e2e_test_race.sh index d66a427a5ff..32374a25c17 100755 --- a/tools/e2e_test_race.sh +++ b/tools/e2e_test_race.sh @@ -14,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +source build.env + temp_log_file="$(mktemp --suffix .unit_test_race.log)" trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT -# This can be removed once the docker images are rebuilt -export GO111MODULE=on - # Wrapper around go test -race. # This script exists because the -race test doesn't allow to distinguish @@ -34,6 +33,7 @@ export GO111MODULE=on # All endtoend Go packages with test files. # Output per line: * packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort) +packages_with_tests=$(echo "$packages_with_tests" | grep -vE "go/test/endtoend" | cut -d" " -f1) # endtoend tests should be in a directory called endtoend all_e2e_tests=$(echo "$packages_with_tests" | cut -d" " -f1) diff --git a/tools/e2e_test_runner.sh b/tools/e2e_test_runner.sh index 4f7822d18ef..c581957a366 100755 --- a/tools/e2e_test_runner.sh +++ b/tools/e2e_test_runner.sh @@ -1,13 +1,13 @@ #!/bin/bash # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,6 +29,9 @@ # Set VT_GO_PARALLEL variable in the same way as the Makefile does. # We repeat this here because this script is called directly by test.go # and not via the Makefile. + +source build.env + if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" fi @@ -38,11 +41,11 @@ fi packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort) # Flaky tests have the suffix "_flaky_test.go". -all_except_flaky_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | cut -d" " -f1) -flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1) +all_except_flaky_and_cluster_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1) +flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1) # Run non-flaky tests. -echo "$all_except_flaky_tests" | xargs go test $VT_GO_PARALLEL +echo "$all_except_flaky_and_cluster_tests" | xargs go test $VT_GO_PARALLEL if [ $? -ne 0 ]; then echo "ERROR: Go unit tests failed. See above for errors." echo diff --git a/tools/generate_web_artifacts.sh b/tools/generate_web_artifacts.sh index d21da7e84e6..a7080db7828 100755 --- a/tools/generate_web_artifacts.sh +++ b/tools/generate_web_artifacts.sh @@ -20,7 +20,7 @@ set -e -vtctld2_dir=$VTTOP/web/vtctld2 +vtctld2_dir=$VTROOT/web/vtctld2 if [[ -d $vtctld2_dir/app ]]; then rm -rf $vtctld2_dir/app fi diff --git a/tools/make-release-packages.sh b/tools/make-release-packages.sh new file mode 100755 index 00000000000..6ec36c5e85f --- /dev/null +++ b/tools/make-release-packages.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# This script builds and packages a Vitess release suitable for creating a new +# release on https://github.com/vitessio/vitess/releases. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# sudo gem install --no-ri --no-rdoc fpm + +source build.env + +SHORT_REV="$(git rev-parse --short HEAD)" +VERSION="5.0.0" # tmp +# TODO: We can discover version from the last tag, we just need to have this setup. +# TAG_VERSION="$(git describe --tags --dirty --always | sed s/v//)" + +RELEASE_ID="vitess-${VERSION}-${SHORT_REV}" +RELEASE_DIR="${VTROOT}/releases/${RELEASE_ID}" +DESCRIPTION="A database clustering system for horizontal scaling of MySQL + +Vitess is a database solution for deploying, scaling and managing large +clusters of MySQL instances. It's architected to run as effectively in a public +or private cloud architecture as it does on dedicated hardware. It combines and +extends many important MySQL features with the scalability of a NoSQL database." + +DEB_FILE="vitess_${VERSION}-${SHORT_REV}_amd64.deb" +RPM_FILE="vitess-${VERSION}-${SHORT_REV}.x86_64.rpm" +TAR_FILE="${RELEASE_ID}.tar.gz" + +make tools +make build + +mkdir -p releases + +# Copy a subset of binaries from issue #5421 +mkdir -p "${RELEASE_DIR}/bin" +for binary in vttestserver mysqlctl mysqlctld query_analyzer topo2topo vtaclcheck vtbackup vtbench vtclient vtcombo vtctl vtctlclient vtctld vtexplain vtgate vttablet vtworker vtworkerclient zk zkctl zkctld; do + cp "bin/$binary" "${RELEASE_DIR}/bin/" +done; + +# Copy remaining files, preserving date/permissions +# But resolving symlinks +cp -rpfL {config,vthook,web,examples} "${RELEASE_DIR}/" + +cd "${RELEASE_DIR}/.." +tar -czf "${TAR_FILE}" "${RELEASE_ID}" + +fpm \ + --force \ + --input-type dir \ + --name vitess \ + --version "${VERSION}" \ + --url "https://vitess.io/" \ + --description "${DESCRIPTION}" \ + --license "Apache License - Version 2.0, January 2004" \ + --prefix "/vt" \ + --directories "/vt" \ + --before-install "$VTROOT/tools/preinstall.sh" \ + -C "${RELEASE_DIR}" \ + --package "$(dirname "${RELEASE_DIR}")" \ + --iteration "${SHORT_REV}" \ + -t deb --deb-no-default-config-files + +fpm \ + --force \ + --input-type dir \ + --name vitess \ + --version "${VERSION}" \ + --url "https://vitess.io/" \ + --description "${DESCRIPTION}" \ + --license "Apache License - Version 2.0, January 2004" \ + --prefix "/vt" \ + --directories "/vt" \ + --before-install "$VTROOT/tools/preinstall.sh" \ + -C "${RELEASE_DIR}" \ + --package "$(dirname "${RELEASE_DIR}")" \ + --iteration "${SHORT_REV}" \ + -t rpm + +echo "" +echo "Packages created as of $(date +"%m-%d-%y") at $(date +"%r %Z")" +echo "" +echo "Package | SHA256" +echo "------------ | -------------" +echo "${TAR_FILE} | $(sha256sum "${VTROOT}/releases/${TAR_FILE}" | awk '{print $1}')" +echo "${DEB_FILE} | $(sha256sum "${VTROOT}/releases/${DEB_FILE}" | awk '{print $1}')" +echo "${RPM_FILE} | $(sha256sum "${VTROOT}/releases/${RPM_FILE}" | awk '{print $1}')" diff --git a/tools/preinstall.sh b/tools/preinstall.sh new file mode 100755 index 00000000000..1d9c80ca782 --- /dev/null +++ b/tools/preinstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if ! /usr/bin/getent group vitess >/dev/null ; then + groupadd -r vitess +fi + +if ! /usr/bin/getent passwd vitess >/dev/null ; then + useradd -r -g vitess vitess +fi diff --git a/tools/shell_functions.inc b/tools/shell_functions.inc index 00696dee5fb..bd344db97ef 100644 --- a/tools/shell_functions.inc +++ b/tools/shell_functions.inc @@ -60,3 +60,9 @@ function prepend_path() { # Return path variable unchanged. echo "$1" } + +function fail() { + echo "ERROR: $1" + exit 1 +} + diff --git a/tools/unit_test_race.sh b/tools/unit_test_race.sh index fef312ff340..2253cab77bc 100755 --- a/tools/unit_test_race.sh +++ b/tools/unit_test_race.sh @@ -14,39 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -temp_log_file="$(mktemp --suffix .unit_test_race.log)" -trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT +source build.env -# Wrapper around go test -race. - -# This script exists because the -race test doesn't allow to distinguish -# between a failed (e.g. flaky) unit test and a found data race. -# Although Go 1.5 says 'exit status 66' in case of a race, it exits with 1. -# Therefore, we manually check the output of 'go test' for data races and -# exit with an error if one was found. -# TODO(mberlin): Test all packages (go/... instead of go/vt/...) once -# go/cgzip is moved into a separate repository. We currently -# skip the cgzip package because -race takes >30 sec for it. +if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then + VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" +fi # All Go packages with test files. # Output per line: * +# TODO: This tests ./go/vt/... instead of ./go/... due to a historical reason. +# When https://github.com/vitessio/vitess/issues/5493 is closed, we should change it. + packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/vt/... | sort) # endtoend tests should be in a directory called endtoend all_except_e2e_tests=$(echo "$packages_with_tests" | cut -d" " -f1 | grep -v "endtoend") # Run non endtoend tests. -echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race 2>&1 | tee $temp_log_file -if [ ${PIPESTATUS[0]} -ne 0 ]; then - if grep "WARNING: DATA RACE" -q $temp_log_file; then - echo - echo "ERROR: go test -race found a data race. See log above." - exit 2 - fi - - echo "ERROR: go test -race found NO data race, but failed. See log above." +echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race + +if [ $? -ne 0 ]; then + echo "WARNING: POSSIBLE DATA RACE" + echo + echo "ERROR: go test -race failed. See log above." exit 1 fi - -echo -echo "SUCCESS: No data race was found." diff --git a/tools/unit_test_runner.sh b/tools/unit_test_runner.sh index 382dd2f0b76..4cffb3eb913 100755 --- a/tools/unit_test_runner.sh +++ b/tools/unit_test_runner.sh @@ -29,6 +29,9 @@ # Set VT_GO_PARALLEL variable in the same way as the Makefile does. # We repeat this here because this script is called directly by test.go # and not via the Makefile. + +source build.env + if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" fi diff --git a/vagrant-scripts/bootstrap_vm.sh b/vagrant-scripts/bootstrap_vm.sh index bea07394dea..46df11227b4 100755 --- a/vagrant-scripts/bootstrap_vm.sh +++ b/vagrant-scripts/bootstrap_vm.sh @@ -23,11 +23,10 @@ apt-get install -y make \ python-dev \ python-virtualenv \ python-mysqldb \ + python-pip \ libssl-dev \ g++ \ - mercurial \ git \ - pkg-config \ bison \ curl \ openjdk-8-jdk \ diff --git a/vagrant-scripts/vagrant-bashrc b/vagrant-scripts/vagrant-bashrc index 27b1828450c..49ddf89943c 100644 --- a/vagrant-scripts/vagrant-bashrc +++ b/vagrant-scripts/vagrant-bashrc @@ -9,8 +9,6 @@ then fi ulimit -n 10000 -export MYSQL_FLAVOR=MySQL56 -export VT_MYSQL_ROOT=/usr # This is just to make sure the vm can write into these directories sudo chown "$(whoami)":"$(whoami)" /vagrant sudo chown "$(whoami)":"$(whoami)" /vagrant/src