Merge remote-tracking branch 'upstream/master' into morgo-fix-etcd34

Signed-off-by: Morgan Tocker <[email protected]>

.github/CODEOWNERS

@@ -2,3 +2,5 @@
 
 /docker/ @derekperkins @dkhenry
 /helm/ @derekperkins @dkhenry
+/config/mycnf/ @morgo
+/go/vt/mysqlctl/mysqld.go @morgo

.github/workflows/check_make_parser.yml

@@ -0,0 +1,37 @@
+name: check_make_parser
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: check_make_parser
+      run: |
+        export PATH=$PWD/bin:$PATH
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/check_make_parser.sh
+

.github/workflows/cluster_endtoend.yml

@@ -0,0 +1,41 @@
+name: cluster_endtoend
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        GOBIN=$PWD/bin make build
+
+    - name: cluster_endtoend
+      run: |
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_cluster

.github/workflows/e2e_race.yml

@@ -0,0 +1,41 @@
+name: e2e_race
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        GOBIN=$PWD/bin make build
+
+    - name: e2e_race
+      run: |
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_race

.github/workflows/e2e-test-cluster.yml → .github/workflows/endtoend.yml

@@ -1,4 +1,4 @@
-name: e2e Test Cluster
+name: endtoend
 on: [push, pull_request]
 jobs:
 
@@ -34,8 +34,9 @@ jobs:
       run: |
         GOBIN=$PWD/bin make build
 
-    - name: Run e2e test cluster
+    - name: endtoend
       run: |
         export PATH=$PWD/bin:$PATH
         source ./dev.env
-        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/e2e_test_cluster.sh
+        mkdir -p /tmp/vtdataroot
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/e2e_test_runner.sh

.github/workflows/local-example.yml → .github/workflows/local_example.yml

@@ -1,4 +1,4 @@
-name: Local Example
+name: local_example
 on: [push, pull_request]
 jobs:
 
@@ -34,7 +34,7 @@ jobs:
       run: |
         GOBIN=$PWD/bin make build
 
-    - name: Run Local Example
+    - name: local_example
       run: |
         export PATH=$PWD/bin:$PATH
         VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD test/local_example.sh

.github/workflows/unit.yml

@@ -0,0 +1,40 @@
+name: unit
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.12
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget ant openjdk-8-jdk
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        VTTOP=$PWD VTROOT=$PWD BUILD_PYTHON=0 ./bootstrap.sh
+
+    - name: Build
+      run: |
+        GOBIN=$PWD/bin make build
+
+    - name: unit
+      run: |
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        mkdir -p /tmp/vtdataroot
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/unit_test_runner.sh

.github/workflows/unit_race.yml

@@ -0,0 +1,41 @@
+name: unit_race
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        GOBIN=$PWD/bin make build
+
+    - name: unit_race
+      run: |
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make unit_test_race

config/init_db.sql

@@ -84,6 +84,13 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
   SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
   ON *.* TO 'vt_filtered'@'localhost';
 
+# User for general MySQL monitoring.
+CREATE USER 'vt_monitoring'@'localhost';
+GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
+  ON *.* TO 'vt_monitoring'@'localhost';
+GRANT SELECT, UPDATE, DELETE, DROP
+  ON performance_schema.* TO 'vt_monitoring'@'localhost';
+
 # User for Orchestrator (https://github.com/github/orchestrator).
 CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';
 GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD

config/mycnf/default-fast.cnf

@@ -1,37 +1,24 @@
-# basic config parameters for all db instances in the grid
+# This sets some unsafe settings specifically for 
+# the test-suite which is currently MySQL 5.7 based
+# In future it should be renamed testsuite.cnf
 
-sql_mode = STRICT_TRANS_TABLES
-character_set_server = utf8
-collation_server = utf8_general_ci
-connect_timeout = 30
-datadir = {{.DataDir}}
-expire_logs_days = 3
-innodb_buffer_pool_size = 64M
-innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_flush_log_at_trx_commit = 2
-innodb_lock_wait_timeout = 20
+innodb_buffer_pool_size = 32M
+innodb_flush_log_at_trx_commit = 0
 innodb_log_buffer_size = 1M
-innodb_log_file_size = 4M
-innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
+innodb_log_file_size = 5M
+
+# Native AIO tends to run into aio-max-nr limit during test startup.
+innodb_use_native_aio = 0
+
 key_buffer_size = 2M
-log-error = {{.ErrorLogPath}}
-long_query_time = 2
-pid-file = {{.PidFile}}
-port = {{.MysqlPort}}
-# all db instances should start in read-only mode - once the db is started and
-# fully functional, we'll push it into read-write mode
-read-only
-server-id = {{.ServerID}}
-skip-name-resolve
-# we now need networking for replication. this is a tombstone to simpler times.
-#skip_networking
-# all db instances should skip the slave startup - that way we can do any
-# out-of-bounds checking before we restart everything - in case we need to do
-# some extra work to skip mangled transactions or fudge the slave start
-skip_slave_start
-slave_net_timeout = 60
-slave_load_tmpdir = {{.SlaveLoadTmpDir}}
-slow-query-log
-slow-query-log-file = {{.SlowLogPath}}
-socket = {{.SocketFile}}
-tmpdir = {{.TmpDir}}
+sync_binlog=0
+innodb_doublewrite=0
+
+# These two settings are required for the testsuite to pass, 
+# but enabling them does not spark joy. They should be removed
+# in the future. See:
+# https://github.com/vitessio/vitess/issues/5395
+# https://github.com/vitessio/vitess/issues/5396
+
+binlog-format=statement
+sql_mode = STRICT_TRANS_TABLES

config/mycnf/default.cnf

@@ -1,35 +1,35 @@
-# basic config parameters for all db instances in the grid
+# Global configuration that is auto-included for all MySQL/MariaDB versions
 
-sql_mode = STRICT_TRANS_TABLES
-binlog_format = statement
-character_set_server = utf8
-collation_server = utf8_general_ci
-connect_timeout = 30
 datadir = {{.DataDir}}
-expire_logs_days = 3
-innodb_buffer_pool_size = 32M
 innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_flush_log_at_trx_commit = 2
-innodb_lock_wait_timeout = 20
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
 log-error = {{.ErrorLogPath}}
-long_query_time = 2
-max_allowed_packet = 64M
-max_connections = 500
 pid-file = {{.PidFile}}
 port = {{.MysqlPort}}
+
 # all db instances should start in read-only mode - once the db is started and
 # fully functional, we'll push it into read-write mode
 read-only
 server-id = {{.ServerID}}
-skip-name-resolve
+
 # all db instances should skip the slave startup - that way we can do any
 # additional configuration (like enabling semi-sync) before we connect to
 # the master.
 skip_slave_start
-slave_net_timeout = 60
 slave_load_tmpdir = {{.SlaveLoadTmpDir}}
-slow-query-log
-slow-query-log-file = {{.SlowLogPath}}
 socket = {{.SocketFile}}
 tmpdir = {{.TmpDir}}
+
+slow-query-log-file = {{.SlowLogPath}}
+
+# These are sensible defaults that apply to all MySQL/MariaDB versions
+
+long_query_time = 2
+slow-query-log
+skip-name-resolve
+connect_timeout = 30
+innodb_lock_wait_timeout = 20
+max_allowed_packet = 64M
+max_connections = 500
+
+