Merge pull request #163 from tinyspeck/slack-vitess-release-merge

6.0 Slack vitess release merge (w/consul 1.5.1, no k3s)

.dockerignore

@@ -6,7 +6,6 @@ java/*/bin
 php/vendor
 releases
 /dist/
-/py-vtdb/
 /vthook/
 /bin/
 /vtdataroot/

.github/workflows/check_formatting.yml

@@ -0,0 +1,21 @@
+name: check_formatting
+on: [pull_request]
+jobs:
+
+  build:
+    name: Check Formatting
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run go fmt
+      run: |
+        gofmt -l . | grep -vF vendor/ && exit 1 || echo "All files formatted correctly"
+

.github/workflows/check_make_visitor.yml

@@ -0,0 +1,35 @@
+name: check_make_visitor
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Check Make Visitor
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run make minimaltools
+      run: |
+        make minimaltools
+
+    - name: check_make_visitor
+      run: |
+        misc/git/hooks/visitorgen
+

.github/workflows/cluster_endtoend.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
 
     steps:
     - name: Set up Go
@@ -32,7 +32,16 @@ jobs:
         sudo apt-get update
         sudo apt-get install percona-xtrabackup-24
 
+    - name: Installing zookeeper and consul
+      run: |
+        # Only running for shard 18 and 24 where we need to install consul and zookeeper
+        if [[ ${{matrix.name}} == 18 || ${{matrix.name}} == 24 ]]; then
+          make tools
+        fi
+
     - name: sharded cluster_endtoend
+      timeout-minutes: 30
       run: |
         source build.env
-        eatmydata -- go run test.go -docker=false -print-log -shard ${{matrix.name}} 
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard ${{matrix.name}}
+        

.github/workflows/cluster_initial_sharding_multi.yml

@@ -0,0 +1,20 @@
+name: cluster_initial_sharding_multi
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: cluster initial sharding multi
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run initial sharding multi
+      run: |
+        go run test.go -print-log initial_sharding_multi

.github/workflows/cluster_vtctl_web.yml

@@ -0,0 +1,22 @@
+name: cluster_vtctl_web
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: cluster vtctl web
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run vtctl web
+      run: |
+        # Running web test inside docker
+        go run test.go -docker=true -print-log -shard 10
+

.github/workflows/create_release.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.12
+        go-version: 1.13
 
     - name: Check out code
       uses: actions/checkout@v2
@@ -26,7 +26,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y make ruby ruby-dev
         go mod download
-        sudo gem install --no-ri --no-rdoc fpm
+        sudo gem install fpm
 
     - name: Make Packages
       run: |

.github/workflows/e2e_race.yml

@@ -30,5 +30,6 @@ jobs:
         make minimaltools
 
     - name: e2e_race
+      timeout-minutes: 30
       run: |
         make e2e_test_race

.github/workflows/endtoend.yml

@@ -34,5 +34,6 @@ jobs:
         make build
 
     - name: endtoend
+      timeout-minutes: 30
       run: |
         tools/e2e_test_runner.sh

.github/workflows/legacy_local_example.yml

@@ -0,0 +1,54 @@
+name: local_example
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Legacy local example using ${{ matrix.topo }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        topo: [etcd,k8s]
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        if [ ${{matrix.os}} = "ubuntu-latest" ]; then
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        elif [ ${{matrix.os}} = "macos-latest" ]; then
+          brew install [email protected] make unzip etcd curl git wget
+        fi
+        go mod download
+
+    - name: Run make minimaltools
+      run: |
+        make minimaltools
+
+    - name: Build
+      run: |
+        make build
+
+    - name: local_example
+      timeout-minutes: 30
+      run: |
+        export TOPO=${{matrix.topo}}
+        if [ ${{matrix.os}} = "macos-latest" ]; then
+          export PATH="/usr/local/opt/[email protected]/bin:$PATH"
+        fi
+        # Make sure that testing is entirely non-reliant on config
+        mv config config-moved
+        eatmydata -- test/legacy_local_example.sh

.github/workflows/local_example.yml

@@ -42,6 +42,7 @@ jobs:
         make build
 
     - name: local_example
+      timeout-minutes: 30
       run: |
         if [ ${{matrix.os}} = "macos-latest" ]; then
           export PATH="/usr/local/opt/[email protected]/bin:$PATH"

.github/workflows/misc_test_docker.yml

@@ -0,0 +1,20 @@
+name: misc test
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Misc Test
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run Misc test which requires docker
+      run: |
+        go run test.go -docker=true -shard 25

.github/workflows/sonar_analysis.yml

@@ -0,0 +1,53 @@
+name: sonar_analysis
+on:
+  push:
+    branches:
+      - 'sonartest'
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          go mod download
+
+      - name: Execute unit test and cluster endtoend test
+        run: |
+          eatmydata -- ./tools/all_test_for_coverage.sh
+          mkdir report
+          cp /tmp/*.out ./report/.
+
+      - name: Analyse sonar
+        run: |
+          export SONAR_SCANNER_VERSION=4.2.0.1873
+          export SONAR_SCANNER_HOME=$HOME/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux
+          curl --create-dirs -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip
+          unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/
+          export PATH=$SONAR_SCANNER_HOME/bin:$PATH
+          export SONAR_SCANNER_OPTS="-server"
+
+          sonar-scanner \
+            -Dsonar.projectKey=vitessio \
+            -Dsonar.organization=vitess \
+            -Dsonar.host.url=https://sonarcloud.io \
+            -Dsonar.login=${SONAR_TOKEN} \
+            -Dsonar.go.coverage.reportPaths=report/*.out
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/unit.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        name: [percona56, mysql57, mariadb101, mariadb102, mariadb103]
+        name: [percona56, mysql57, mysql80, mariadb101, mariadb102, mariadb103]
 
     steps:
     - name: Set up Go
@@ -19,14 +19,21 @@ jobs:
 
     - name: Get dependencies
       run: |
+        export DEBIAN_FRONTEND="noninteractive"
         sudo apt-get update
 
         if [ ${{matrix.name}} = "mysql57" ]; then
           sudo apt-get install -y mysql-server mysql-client
         else
           # Uninstall likely installed MySQL first
-          sudo apt-get remove -y mysql-server mysql-client
-          
+          sudo systemctl stop apparmor
+          sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+          sudo apt-get -y autoremove
+          sudo apt-get -y autoclean
+          sudo deluser mysql
+          sudo rm -rf /var/lib/mysql
+          sudo rm -rf /etc/mysql
+ 
           if [ ${{matrix.name}} = "percona56" ]; then
             sudo rm -rf /var/lib/mysql
             sudo apt install -y gnupg2
@@ -41,7 +48,11 @@ jobs:
             sudo apt-get update
             sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
           elif [ ${{matrix.name}} = "mariadb101" ]; then
-            sudo apt install -y mariadb-server mariadb-client
+            sudo apt-get install -y software-properties-common
+            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu bionic main'
+            sudo apt update
+            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
           elif [ ${{matrix.name}} = "mariadb102" ]; then
             sudo apt-get install -y software-properties-common
             sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
@@ -74,5 +85,6 @@ jobs:
         make tools
 
     - name: unit
+      timeout-minutes: 30
       run: |
-        eatmydata -- make test
+        eatmydata -- make unit_test

.github/workflows/unit_race.yml

@@ -34,5 +34,6 @@ jobs:
         make tools
 
     - name: unit_race
+      timeout-minutes: 30
       run: |
         eatmydata -- make unit_test_race

.gitignore

@@ -30,9 +30,6 @@ tags
 # C build dirs
 **/build
 
-# site-local example files
-/examples/kubernetes/config.sh
-
 # generated protobuf files
 /go/vt/.proto.tmp
 
@@ -81,7 +78,10 @@ releases
 .vagrant
 
 /dist/
-/py-vtdb
 /vthook/
 /bin/
 /vtdataroot/
+venv
+
+.scannerwork
+report