Slack sync 2019 01 30

bootstrap.sh

@@ -23,6 +23,7 @@
 # 3. Detection of installed MySQL and setting MYSQL_FLAVOR.
 # 4. Installation of development related steps e.g. creating Git hooks.
 
+BUILD_TESTS=${BUILD_TESTS:-1}
 
 #
 # 0. Initialization and helper methods.
@@ -45,25 +46,36 @@ function fail() {
 [[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory"
 
 go version &>/dev/null  || fail "Go is not installed or is not on \$PATH"
+[[ "$(go version 2>&1)" =~ go1\.[1-9][1-9] ]] || fail "Go is not version 1.11+"
 
 # Set up the proper GOPATH for go get below.
-source ./dev.env
+if [ "$BUILD_TESTS" == 1 ] ; then
+    source ./dev.env
+else
+    source ./build.env
+fi
 
 # Create main directories.
 mkdir -p "$VTROOT/dist"
 mkdir -p "$VTROOT/bin"
 mkdir -p "$VTROOT/lib"
 mkdir -p "$VTROOT/vthook"
 
-# Set up required soft links.
-# TODO(mberlin): Which of these can be deleted?
-ln -snf "$VTTOP/config" "$VTROOT/config"
-ln -snf "$VTTOP/data" "$VTROOT/data"
-ln -snf "$VTTOP/py" "$VTROOT/py-vtdb"
-ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
-ln -snf "$VTTOP/test/vthook-test.sh" "$VTROOT/vthook/test.sh"
-ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
-ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
+if [ "$BUILD_TESTS" == 1 ] ; then
+    # Set up required soft links.
+    # TODO(mberlin): Which of these can be deleted?
+    ln -snf "$VTTOP/config" "$VTROOT/config"
+    ln -snf "$VTTOP/data" "$VTROOT/data"
+    ln -snf "$VTTOP/py" "$VTROOT/py-vtdb"
+    ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+    ln -snf "$VTTOP/test/vthook-test.sh" "$VTROOT/vthook/test.sh"
+    ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
+    ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
+else
+    ln -snf "$VTTOP/config" "$VTROOT/config"
+    ln -snf "$VTTOP/data" "$VTROOT/data"
+    ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+fi
 
 # install_dep is a helper function to generalize the download and installation of dependencies.
 #
@@ -136,8 +148,10 @@ function install_grpc() {
   grpcio_ver=$version
   $PIP install --upgrade grpcio=="$grpcio_ver" grpcio-tools=="$grpcio_ver"
 }
-install_dep "gRPC" "1.16.0" "$VTROOT/dist/grpc" install_grpc
 
+if [ "$BUILD_TESTS" == 1 ] ; then
+    install_dep "gRPC" "1.16.0" "$VTROOT/dist/grpc" install_grpc
+fi
 
 # Install protoc.
 function install_protoc() {
@@ -225,8 +239,9 @@ function install_pymock() {
   popd >/dev/null
 }
 pymock_version=1.0.1
-install_dep "py-mock" "$pymock_version" "$VTROOT/dist/py-mock-$pymock_version" install_pymock
-
+if [ "$BUILD_TESTS" == 1 ] ; then
+    install_dep "py-mock" "$pymock_version" "$VTROOT/dist/py-mock-$pymock_version" install_pymock
+fi
 
 # Download Selenium (necessary to run test/vtctld_web_test.py).
 function install_selenium() {
@@ -239,7 +254,9 @@ function install_selenium() {
   # instead of go/dist/selenium/lib/python3.5/site-packages and then can't find module 'pip._vendor.requests'
   PYTHONPATH='' $PIP install selenium
 }
-install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
+if [ "$BUILD_TESTS" == 1 ] ; then
+    install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
+fi
 
 
 # Download chromedriver (necessary to run test/vtctld_web_test.py).
@@ -251,7 +268,9 @@ function install_chromedriver() {
   unzip -o -q chromedriver_linux64.zip -d "$dist"
   rm chromedriver_linux64.zip
 }
-install_dep "chromedriver" "2.44" "$VTROOT/dist/chromedriver" install_chromedriver
+if [ "$BUILD_TESTS" == 1 ] ; then
+    install_dep "chromedriver" "2.44" "$VTROOT/dist/chromedriver" install_chromedriver
+fi
 
 
 #
@@ -300,47 +319,52 @@ govendor sync || fail "Failed to download/update dependencies with govendor. Ple
 
 
 # find mysql and prepare to use libmysqlclient
-if [ -z "$MYSQL_FLAVOR" ]; then
-  export MYSQL_FLAVOR=MySQL56
-  echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
-fi
-case "$MYSQL_FLAVOR" in
-  "MySQL56")
-    myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
-    [[ "$myversion" =~ Distrib\ 5\.[67] || "$myversion" =~ Ver\ 8\. ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
-    echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
-    ;;
-
-  "MariaDB")
-    myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
-    [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
-    echo "Found MariaDB installation in $VT_MYSQL_ROOT."
-    ;;
-
-  *)
-    fail "Unsupported MYSQL_FLAVOR $MYSQL_FLAVOR"
-    ;;
 
-esac
-
-# save the flavor that was used in bootstrap, so it can be restored
-# every time dev.env is sourced.
-echo "$MYSQL_FLAVOR" > "$VTROOT/dist/MYSQL_FLAVOR"
+if [ "$BUILD_TESTS" == 1 ] ; then
+  if [ -z "$MYSQL_FLAVOR" ]; then
+    export MYSQL_FLAVOR=MySQL56
+    echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
+  fi
+  case "$MYSQL_FLAVOR" in
+    "MySQL56")
+      myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
+      [[ "$myversion" =~ Distrib\ 5\.[67] || "$myversion" =~ Ver\ 8\. ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
+      echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
+      ;;
+
+    "MariaDB")
+      myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
+      [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
+      echo "Found MariaDB installation in $VT_MYSQL_ROOT."
+      ;;
+
+    *)
+      fail "Unsupported MYSQL_FLAVOR $MYSQL_FLAVOR"
+      ;;
 
+  esac
+  # save the flavor that was used in bootstrap, so it can be restored
+  # every time dev.env is sourced.
+  echo "$MYSQL_FLAVOR" > "$VTROOT/dist/MYSQL_FLAVOR"
+fi
 
 #
 # 4. Installation of development related steps e.g. creating Git hooks.
 #
 
-
-# Create the Git hooks.
-echo "creating git hooks"
-mkdir -p "$VTTOP/.git/hooks"
-ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
-ln -sf "$VTTOP/misc/git/prepare-commit-msg.bugnumber" "$VTTOP/.git/hooks/prepare-commit-msg"
-ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
-(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
+if [ "$BUILD_TESTS" == 1 ] ; then
+ # Create the Git hooks.
+ echo "creating git hooks"
+ mkdir -p "$VTTOP/.git/hooks"
+ ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
+ ln -sf "$VTTOP/misc/git/prepare-commit-msg.bugnumber" "$VTTOP/.git/hooks/prepare-commit-msg"
+ ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
+ (cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
+ echo
+ echo "bootstrap finished - run 'source dev.env' in your shell before building."
+else
+ echo
+ echo "bootstrap finished - run 'source build.env' in your shell before building."    
+fi
 
 
-echo
-echo "bootstrap finished - run 'source dev.env' in your shell before building."

build.env

@@ -0,0 +1,39 @@
+# No shebang line as this script is sourced from an external shell.
+
+# Copyright 2017 Google Inc.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Plese ensure dev.env is written in a way which is POSIX (bourne)
+# shell compatible.
+# - Some build systems like rpm require the different scriptlets used
+#   to build a package to be run under a POSIX shell so non-POSIX
+#   syntax will break that as dev.env will not be sourced by bash..
+
+# Import prepend_path function.
+dir="$(dirname "${BASH_SOURCE[0]}")"
+# shellcheck source=tools/shell_functions.inc
+if ! source "${dir}/tools/shell_functions.inc"; then
+  echo "failed to load tools/shell_functions.inc"
+  return 1
+fi
+
+VTTOP=$(pwd)
+export VTTOP
+VTROOT="${VTROOT:-${VTTOP/\/src\/vitess.io\/vitess/}}"
+export VTROOT
+# VTTOP sanity check
+if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then
+  echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess"
+fi
+

config/init_db.sql

@@ -84,3 +84,5 @@ GRANT SELECT
 
 FLUSH PRIVILEGES;
 
+RESET SLAVE ALL;
+RESET MASTER;

dev.env

@@ -20,22 +20,8 @@
 #   to build a package to be run under a POSIX shell so non-POSIX
 #   syntax will break that as dev.env will not be sourced by bash..
 
-# Import prepend_path function.
-dir="$(dirname "${BASH_SOURCE[0]}")"
-# shellcheck source=tools/shell_functions.inc
-if ! source "${dir}/tools/shell_functions.inc"; then
-  echo "failed to load tools/shell_functions.inc"
-  return 1
-fi
+source ./build.env
 
-VTTOP=$(pwd)
-export VTTOP
-VTROOT="${VTROOT:-${VTTOP/\/src\/vitess.io\/vitess/}}"
-export VTROOT
-# VTTOP sanity check
-if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then
-  echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess"
-fi
 export GOTOP=$VTTOP/go
 export PYTOP=$VTTOP/py
 

doc/FAQ.md

@@ -20,7 +20,7 @@ If no tablet type was specified, then VTGate chooses its default, which can be o
 
 Vitess supports different modes. In OLTP mode, the result size is typically limited to a preset number (10,000 rows by default). This limit can be adjusted based on your needs.
 
-However, OLAP mode has no limit to the number of rows returned. In order to change to this mode, you may issue the following command command before executing your query:
+However, OLAP mode has no limit to the number of rows returned. In order to change to this mode, you may issue the following command before executing your query:
 
 ```
 set workload='olap'
@@ -32,7 +32,7 @@ The general convention is to send OLTP queries to `REPLICA` tablet types, and OL
 
 ## Is there a list of supported/unsupported queries?
 
-The list of unsupported constructs is currently in the form of test cases contained in this [test file](https://github.com/vitessio/vitess/blob/master/data/test/vtgate/unsupported_cases.txt). However, contrary to the test cases, there is limited support for SET, DDL and DBA constructs. This will be documented soon.
+The list of unsupported constructs is currently in the form of test cases contained in this [test file](https://github.com/vitessio/vitess/blob/master/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt). However, contrary to the test cases, there is limited support for SET, DDL and DBA constructs. This will be documented soon.
 
 
 ## If I have a log of all queries from my app. Is there a way I can try them against vitess to see how they’ll work?

doc/HorizontalReshardingWorkflowGuide.md

@@ -132,7 +132,7 @@ on the clicked button. The approval buttons are cleared after the phase has
 finished. The next phase will only starts if its previous phase has finished
 successfully.
 
-If the workflow is restored from a checkpoint, you will still see the the
+If the workflow is restored from a checkpoint, you will still see the
 approval button with approved message when there are running tasks under this
 approval. But you don't need to approve the same tasks again for a restarted
 workflow.

doc/Monitoring.md

@@ -22,11 +22,11 @@ Scraping Vitess variables is a good way to integrate Vitess into an existing mon
 
 Vitess also includes support for push-based metrics systems via plug-ins. Each Vitess component would need to be run with the `--emit_stats` flag.
 
-By default, the stats_emit_period is 60s, so each component will push stats to the the selected backend every minute. This is configurable via the `--stats_emit_period` flag.
+By default, the stats_emit_period is 60s, so each component will push stats to the selected backend every minute. This is configurable via the `--stats_emit_period` flag.
 
-Vitess has preliminary plug-ins to support InfluxDB and OpenTSDB as push-based metrics backends. However, there is very limited support at this time, as InfluxDB itself is going through various API breaking changes.
+Vitess has preliminary plug-ins to support OpenTSDB as a push-based metrics backend.
 
-It should be fairly straightforward to write your own plug-in, if you want to support a different backend. The plug-in package simply needs to implement the `PushBackend` interface of the `stats` package. For an example, you can see the [InfluxDB plugin](https://github.com/vitessio/vitess/blob/master/go/stats/influxdbbackend/influxdb_backend.go).
+It should be fairly straightforward to write your own plug-in, if you want to support a different backend. The plug-in package simply needs to implement the `PushBackend` interface of the `stats` package. For an example, you can see the [OpenTSDB plugin](https://github.com/vitessio/vitess/blob/master/go/stats/opentsdb/opentsdb.go).
 
 Once you’ve written the backend plug-in, you also need to register the plug-in from within all the relevant Vitess binaries. An example of how to do this can be seen in [this pull request](https://github.com/vitessio/vitess/pull/469).
 
@@ -36,7 +36,7 @@ Connecting Vitess to a push-based metrics system can be useful if you’re alrea
 
 ## Monitoring with Kubernetes
 
-The existing methods for integrating metrics are not supported in a Kubernetes environment by the Vitess team yet, but are on the roadmap for the future. However, it should be possible to get the InfluxDB backend working with Kubernetes, similar to how [Heapster for Kubernetes works](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/cluster/addons/cluster-monitoring). 
+The existing methods for integrating metrics are not supported in a Kubernetes environment by the Vitess team yet, but are on the roadmap for the future. However, it should be possible to get the Prometheus backend working with Kubernetes, similar to how [Heapster for Kubernetes works](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/prometheus). 
 
 In the meantime, if you run into issues or have questions, please post on our [forum](https://groups.google.com/forum/#!forum/vitess).
 

doc/ScalingMySQL.md

@@ -54,7 +54,7 @@ Setting up these components directly -- for example, writing your own topology s
 * *Recommended*.  Vitess has basic support for identifying or changing a master, but it doesn't aim to fully address this feature. As such, we recommend using another program, like [Orchestrator](https://github.com/github/orchestrator), to monitor the health of your servers and to change your master database when necessary. (In a sharded database, each shard has a master.)
 
 
-* *Recommended*. You should have a way to monitor your database topology and set up alerts as needed. Vitess components facilitate this monitoring by exporting a lot of runtime variables, like QPS over the last few minutes, error rates, and query latency. The variables are exported in JSON format, and Vitess also supports an InfluxDB plug-in.
+* *Recommended*. You should have a way to monitor your database topology and set up alerts as needed. Vitess components facilitate this monitoring by exporting a lot of runtime variables, like QPS over the last few minutes, error rates, and query latency. The variables are exported in JSON format, and Vitess also supports a Prometheus plug-in.
 
 
 * *Optional*. Using the Kubernetes scripts as a base, you could run Vitess components with other configuration management systems (like Puppet) or frameworks (like Mesos or AWS images).

doc/SchemaManagement.md

@@ -64,7 +64,7 @@ ValidateSchemaShard user/0
 
 The <code>[ValidateSchemaKeyspace]({% link reference/vtctl.md %}#validateschemakeyspace)</code>
 command confirms that all of the tablets in a given keyspace have
-the the same schema as the master tablet on shard <code>0</code>
+the same schema as the master tablet on shard <code>0</code>
 in that keyspace. Thus, whereas the <code>ValidateSchemaShard</code>
 command confirms the consistency of the schema on tablets within a shard
 for a given keyspace, <code>ValidateSchemaKeyspace</code> confirms the

doc/ServerConfiguration.md

@@ -71,7 +71,7 @@ This rule is not strictly enforced. You are allowed to add these things, but at
 
 Similar guidelines should be used when deciding to bypass Vitess to send statements directly to MySQL.
 
-Vitess also requires you to turn on STRICT_TRANS_TABLES mode. Otherwise, it cannot accurately predict what will be written to the database.
+Vitess also requires you to turn on STRICT_TRANS_TABLES or STRICT_ALL_TABLES mode. Otherwise, it cannot accurately predict what will be written to the database.
 
 It’s safe to apply backward compatible DDLs directly to MySQL. VTTablets can be configured to periodically check the schema for changes.
 

doc/Upgrading.md

@@ -2,7 +2,7 @@
 
 This document highlights things to look after when upgrading a Vitess production installation to a newer Vitess release.
 
-Generally speaking, upgrading Vitess is a safe and and easy process because it is explicitly designed for it. This is because in YouTube we follow the practice of releasing new versions often (usually from the tip of the Git master branch).
+Generally speaking, upgrading Vitess is a safe and easy process because it is explicitly designed for it. This is because in YouTube we follow the practice of releasing new versions often (usually from the tip of the Git master branch).
 
 ## Compatibility
 

doc/V3HighLevelDesign.md

@@ -1311,7 +1311,7 @@ When two nodes are grouped, the current join condition becomes the root of the n
 * If it’s a JOIN, the new property is the more restrictive of the two nodes. So, if one of them is a Route, then the new node is also a Route.
 * For a LEFT JOIN, the new property is the same as the LHS node.
 
-If the grouping conditions are not met, then the node remains a join node. In this case, we have to see if the ON clause conditions can be pushed down into the left and/or right nodes. By the fact that the current join is split into two, the ON clause cannot be be pushed as is. Instead, we use associativity rules to our benefit and merge the ON clause conditions into the WHERE clauses of the underlying nodes. The rules are the same as the ones described for a normal WHERE clause.
+If the grouping conditions are not met, then the node remains a join node. In this case, we have to see if the ON clause conditions can be pushed down into the left and/or right nodes. By the fact that the current join is split into two, the ON clause cannot be pushed as is. Instead, we use associativity rules to our benefit and merge the ON clause conditions into the WHERE clauses of the underlying nodes. The rules are the same as the ones described for a normal WHERE clause.
 
 But left joins are slightly different, because the join condition is applied *to the RHS only*. Also, the condition cannot be further pushed into other nested left joins, because they will change the meaning of the statement. For example:
 
@@ -1491,7 +1491,7 @@ If a, b and c where in different groups, the output would be:
 a   b where (b.id=a.id) and (cond1(a.col, b.col))
 ```
 
-The cond2 expression gets pushed into the the where clause for table ‘c’ because it’s the right-most group that’s referenced by the condition. External references will be changed to appropriate bind variables by the rewiring phase.
+The cond2 expression gets pushed into the where clause for table ‘c’ because it’s the right-most group that’s referenced by the condition. External references will be changed to appropriate bind variables by the rewiring phase.
 
 *Once VTGate acquires the ability to perform its own filters, should we stop pushing these conditions into the dependent queries and do it ourselves instead? The answer will usually be no. You almost always want to push down filters. This is because it will let the underlying database scan fewer rows, or choose better indexes. The more restrictive the query is, the better.*