murron logger

Makefile

@@ -1,11 +1,11 @@
 # Copyright 2017 Google Inc.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -186,7 +186,7 @@ docker_base_mysql56:
 
 docker_base_mysql80:
 	chmod -R o=g *
-	docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql80 .
+	docker build -f docker/base/Dockerfile.mysql80 -t vitess/base:mysql80 .
 
 docker_base_mariadb:
 	chmod -R o=g *
@@ -206,7 +206,7 @@ docker_base_percona57:
 
 docker_base_percona80:
 	chmod -R o=g *
-	docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona80 .
+	docker build -f docker/base/Dockerfile.percona80 -t vitess/base:percona80 .
 
 # Run "make docker_lite PROMPT_NOTICE=false" to avoid that the script
 # prompts you to press ENTER and confirm that the vitess/base image is not

bootstrap.sh

@@ -195,13 +195,22 @@ function install_etcd() {
   local version="$1"
   local dist="$2"
 
+  case $(uname) in
+    Linux)  local platform=linux; local ext=tar.gz;;
+    Darwin) local platform=darwin; local ext=zip;;
+  esac
+
   download_url=https://github.com/coreos/etcd/releases/download
-  tar_file="etcd-${version}-linux-amd64.tar.gz"
+  file="etcd-${version}-${platform}-amd64.${ext}"
 
-  wget "$download_url/$version/$tar_file"
-  tar xzf "$tar_file"
-  rm "$tar_file"
-  ln -snf "$dist/etcd-${version}-linux-amd64/etcd" "$VTROOT/bin/etcd"
+  wget "$download_url/$version/$file"
+  if [ "$ext" = "tar.gz" ]; then
+    tar xzf "$file"
+  else
+    unzip "$file"
+  fi
+  rm "$file"
+  ln -snf "$dist/etcd-${version}-${platform}-amd64/etcd" "$VTROOT/bin/etcd"
 }
 install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
 
@@ -211,9 +220,14 @@ function install_consul() {
   local version="$1"
   local dist="$2"
 
+  case $(uname) in
+    Linux)  local platform=linux;;
+    Darwin) local platform=darwin;;
+  esac
+
   download_url=https://releases.hashicorp.com/consul
-  wget "${download_url}/${version}/consul_${version}_linux_amd64.zip"
-  unzip "consul_${version}_linux_amd64.zip"
+  wget "${download_url}/${version}/consul_${version}_${platform}_amd64.zip"
+  unzip "consul_${version}_${platform}_amd64.zip"
   ln -snf "$dist/consul" "$VTROOT/bin/consul"
 }
 install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
@@ -325,13 +339,13 @@ if [ "$BUILD_TESTS" == 1 ] ; then
     echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
   fi
   case "$MYSQL_FLAVOR" in
-    "MySQL56")
+    "MySQL56" | "MySQL80")
       myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
       [[ "$myversion" =~ Distrib\ 5\.[67] || "$myversion" =~ Ver\ 8\. ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
       echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
       ;;
 
-    "MariaDB" | "MariaDB103" )
+    "MariaDB" | "MariaDB103")
       myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
       [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
       echo "Found MariaDB installation in $VT_MYSQL_ROOT."

config/init_db.sql

@@ -1,9 +1,16 @@
 # This file is executed immediately after mysql_install_db,
 # to initialize a fresh data directory.
 
-##########################################
+###############################################################################
+# WARNING: This sql is *NOT* safe for production use,
+#          as it contains default well-known users and passwords.
+#          Care should be taken to change these users and passwords
+#          for production.
+###############################################################################
+
+###############################################################################
 # Equivalent of mysql_secure_installation
-##########################################
+###############################################################################
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -17,9 +24,9 @@ DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
 # Remove test database.
 DROP DATABASE IF EXISTS test;
 
-##########################################
+###############################################################################
 # Vitess defaults
-##########################################
+###############################################################################
 
 # Vitess-internal database.
 CREATE DATABASE IF NOT EXISTS _vt;
@@ -28,12 +35,14 @@ CREATE DATABASE IF NOT EXISTS _vt;
 CREATE TABLE IF NOT EXISTS _vt.local_metadata (
   name VARCHAR(255) NOT NULL,
   value VARCHAR(255) NOT NULL,
-  PRIMARY KEY (name)
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
   ) ENGINE=InnoDB;
 CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
   name VARCHAR(255) NOT NULL,
   value MEDIUMBLOB NOT NULL,
-  PRIMARY KEY (name)
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
   ) ENGINE=InnoDB;
 
 # Admin user with all privileges.

config/mycnf/default-fast.cnf

@@ -18,7 +18,7 @@ innodb_flush_log_at_trx_commit = 2
 innodb_flush_method = O_DIRECT
 innodb_lock_wait_timeout = 20
 innodb_log_buffer_size = 1M
-innodb_log_file_size = 1M
+innodb_log_file_size = 4M
 innodb_log_files_in_group = 2
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
 innodb_max_dirty_pages_pct = 75

config/mycnf/master_mysql80.cnf

@@ -0,0 +1,38 @@
+# Options for enabling GTID
+# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+gtid_mode = ON
+log_bin
+log_slave_updates
+enforce_gtid_consistency
+
+# Crash-safe replication settings.
+master_info_repository = TABLE
+relay_log_info_repository = TABLE
+relay_log_purge = 1
+relay_log_recovery = 1
+
+# Native AIO tends to run into aio-max-nr limit during test startup.
+innodb_use_native_aio = 0
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+
+# disable mysqlx
+mysqlx = 0
+
+# 8.0 changes the default auth-plugin to caching_sha2_password
+default_authentication_plugin = mysql_native_password
+secure_file_priv = NULL

doc/ServerConfiguration.md

@@ -258,7 +258,7 @@ VTTablet requires multiple user credentials to perform its tasks. Since it's req
 
 **connection** parameters
 
-* **db\_socket**: The unix socket to connect on. If this is specifed, host and port will not be used.
+* **db\_socket**: The unix socket to connect on. If this is specified, host and port will not be used.
 * **db\_host**: The host name for the tcp connection.
 * **db\_port**: The tcp port to be used with the db\_host.
 * **db\_charset**: Character set. Only utf8 or latin1 based character sets are supported.

doc/TopologyService.md

@@ -451,7 +451,7 @@ all cells to route traffic. Note this is necessary to access the master in
 another cell.
 
 After the extension to two cells, the original topo service contains both the
-global topology data, and the first cell topology data. The more symetrical
+global topology data, and the first cell topology data. The more symmetrical
 configuration we're after would be to split that original service into two: a
 global one that only contains the global data (spread across both cells), and a
 local one to the original cells. To achieve that split:
@@ -471,7 +471,7 @@ local one to the original cells. To achieve that split:
 * Remove all remaining data in the global topology service that are in the old
   local server root.
 
-After this split, the configuration is completely symetrical:
+After this split, the configuration is completely symmetrical:
 
 * a global topology service, with servers in all cells. Only contains global
   topology data about Keyspaces, Shards and VSchema. Typically it has 5 servers

doc/VTTabletModes.md

@@ -36,7 +36,7 @@ Specifically, the absence of a my.cnf file indicates to vttablet that it's conne
 
 Even if a MySQL is remote, you can still make vttablet perform some management functions. They are as follows:
 
-* `-disable_active_reparents`: If this flag is set, then any reparent or slave commands will not be allowed. These are InitShardMaster, PlannedReparent, PlannedReparent, EmergencyReparent, and ReparentTablet. In this mode, you should use the TabletExternallyReparented command to inform vitess of the current master.
+* `-disable_active_reparents`: If this flag is set, then any reparent or slave commands will not be allowed. These are InitShardMaster, PlannedReparent, EmergencyReparent, and ReparentTablet. In this mode, you should use the TabletExternallyReparented command to inform vitess of the current master.
 * `-master_connect_retry`: This value is give to mysql when it connects a slave to the master as the retry duration parameter.
 * `-enable_replication_reporter`: If this flag is set, then vttablet will transmit replica lag related information to the vtgates, which will allow it to balance load better. Additionally, enabling this will also cause vttablet to restart replication if it was stopped. However, it will do this only if -disable_active_reparents was not turned on.
 * `-enable_semi_sync`: This option will automatically enable semi-sync on new replicas as well as on any tablet that transitions into a replica type. This includes the demotion of a master to a replica.

doc/VitessApi.md

@@ -79,11 +79,11 @@ The following table lists the methods in each group and links to more detail abo
 <tr><td class="api-method-summary-group" colspan="2">v3 API (alpha)</td></tr>
 <tr>
 <td><code><a href="#execute">Execute</a></code></td>
-<td>Execute tries to route the query to the right shard. It depends on the query and bind variables to provide enough information in conjonction with the vindexes to route the query.</td>
+<td>Execute tries to route the query to the right shard. It depends on the query and bind variables to provide enough information in conjunction with the vindexes to route the query.</td>
 </tr>
 <tr>
 <td><code><a href="#streamexecute">StreamExecute</a></code></td>
-<td>StreamExecute executes a streaming query based on shards. It depends on the query and bind variables to provide enough information in conjonction with the vindexes to route the query. Use this method if the query returns a large number of rows.</td>
+<td>StreamExecute executes a streaming query based on shards. It depends on the query and bind variables to provide enough information in conjunction with the vindexes to route the query. Use this method if the query returns a large number of rows.</td>
 </tr>
 </table>
 ##Range-based Sharding
@@ -572,7 +572,7 @@ GetSrvKeyspace returns a SrvKeyspace object (as seen by this vtgate). This metho
 ##v3 API (alpha)
 ### Execute
 
-Execute tries to route the query to the right shard. It depends on the query and bind variables to provide enough information in conjonction with the vindexes to route the query.
+Execute tries to route the query to the right shard. It depends on the query and bind variables to provide enough information in conjunction with the vindexes to route the query.
 
 #### Request
 
@@ -604,7 +604,7 @@ Execute tries to route the query to the right shard. It depends on the query and
 
 ### StreamExecute
 
-StreamExecute executes a streaming query based on shards. It depends on the query and bind variables to provide enough information in conjonction with the vindexes to route the query. Use this method if the query returns a large number of rows.
+StreamExecute executes a streaming query based on shards. It depends on the query and bind variables to provide enough information in conjunction with the vindexes to route the query. Use this method if the query returns a large number of rows.
 
 #### Request
 
@@ -839,7 +839,7 @@ Field describes a single column returned by a query
 | <code>org_name</code> <br>string| |
 | <code>column_length</code> <br>uint32| column_length is really a uint32. All 32 bits can be used. |
 | <code>charset</code> <br>uint32| charset is actually a uint16. Only the lower 16 bits are used. |
-| <code>decimals</code> <br>uint32| decimals is actualy a uint8. Only the lower 8 bits are used. |
+| <code>decimals</code> <br>uint32| decimals is actually a uint8. Only the lower 8 bits are used. |
 | <code>flags</code> <br>uint32| flags is actually a uint16. Only the lower 16 bits are used. |
 
 ### query.QueryResult

doc/VitessSequences.md

@@ -87,7 +87,7 @@ Let's start by looking at the MySQL auto-increment feature:
 
 An early design was to use a single unsharded database and a table with an
 auto-increment value to generate new values. However, this has serious
-limitations, in particular throughtput, and storing one entry for each value in
+limitations, in particular throughput, and storing one entry for each value in
 that table, for no reason.
 
 So we decided instead to base sequences on a MySQL table, and use a single value

doc/VitessTransportSecurityModel.md

@@ -55,7 +55,7 @@ When using gRPC transport, Vitess can use the usual TLS security features
   * grpc\_cert, grpc\_key: server cert and key to use.
   * grpc\_ca (optional): client cert chains to trust. If specified, the client
     must use a certificate signed by one ca in the provided file.
-* A Vitess go client can be configured with symetrical parameters to enable TLS:
+* A Vitess go client can be configured with symmetrical parameters to enable TLS:
   * ...\_grpc\_ca: list of server cert signers to trust.
   * ...\_grpc\_server\_name: name of the server cert to trust, instead of the
     hostname used to connect.

doc/meetups_notes/06-14-2018.md

@@ -56,7 +56,7 @@ Meeting started at 8am PDT on Zoom.
 
 ### New Vitess Logo / Logo Changes
 
-We kicked off the meeting talking about the first agenda item, *Vitess's new logo*. The new logo was mooted in the first place because of its current similarites to [Vivid IT Corp](https://www.vividitcorp.com/).
+We kicked off the meeting talking about the first agenda item, *Vitess's new logo*. The new logo was mooted in the first place because of its current similarities to [Vivid IT Corp](https://www.vividitcorp.com/).
 
 Based on [logos](https://docs.google.com/forms/d/e/1FAIpQLScp5hGY98vpRMxs3oRT8c-XJ_b04ei6uCFiYiQe3nDunFbyuw/viewform) created with CNCF's help, logos 1 and 4 got the highest no of votes.
 
@@ -124,7 +124,7 @@ We are looking for 5 engineers. Job scope at [our site](https://planetscale.com/
 
 We discussed that we need to work on / update official roadmap.
 
-Question from Ameet about whether there's anything on ther roadmap for online backups, because increasingly backups take more time.
+Question from Ameet about whether there's anything on the roadmap for online backups, because increasingly backups take more time.
 
 Sugu says there are 500 lines of code re backup. But we will discuss this in our roadmap discussions.
 

doc/vtctlReference.md

@@ -530,7 +530,7 @@ Validates that all nodes reachable from the specified keyspace are consistent.
 
 ### WaitForDrain
 
-Blocks until no new queries were observed on all tablets with the given tablet type in the specifed keyspace.  This can be used as sanity check to ensure that the tablets were drained after running vtctl MigrateServedTypes  and vtgate is no longer using them. If -timeout is set, it fails when the timeout is reached.
+Blocks until no new queries were observed on all tablets with the given tablet type in the specified keyspace.  This can be used as sanity check to ensure that the tablets were drained after running vtctl MigrateServedTypes  and vtgate is no longer using them. If -timeout is set, it fails when the timeout is reached.
 
 #### Example
 

docker/bootstrap/Dockerfile.mariadb

@@ -1,12 +1,18 @@
 FROM vitess/bootstrap:common
 
 # Install MariaDB 10
-RUN apt-get update \
+RUN apt-get update -y \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
       bzip2 \
       mariadb-server \
       libmariadbclient-dev \
- && rm -rf /var/lib/apt/lists/*
+      libdbd-mysql-perl \
+      rsync \
+      libev4 \
+ && rm -rf /var/lib/apt/lists/* \
+ && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
+ && dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
+ && rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql56

@@ -3,9 +3,12 @@ FROM vitess/bootstrap:common
 # Install MySQL 5.6
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.sks-keyservers.net 5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+    dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+    rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql57

@@ -4,9 +4,12 @@ FROM vitess/bootstrap:common
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \
     for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+    dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+    rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql80

@@ -1,16 +1,19 @@
 FROM vitess/bootstrap:common
 
-# Install MySQL 5.7
+# Install MySQL 8.0
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-8.0.4/binary/debian/stretch/x86_64/percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
+    dpkg -i percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
+    rm -f percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
 
-ENV MYSQL_FLAVOR MySQL56
+ENV MYSQL_FLAVOR MySQL80
 USER vitess
 RUN ./bootstrap.sh