slack vitess v10.pre

.github/CODEOWNERS

@@ -1,33 +1,44 @@
-* @sougou
-
+bootstrap.sh @deepthi
 /config/mycnf/ @askdba @shlomi-noach
 /docker/ @derekperkins @dkhenry
 /examples/compose @shlomi-noach
+/examples/demo @sougou
+/examples/legacy_local @deepthi
 /examples/local @rohit-nayak-ps
+/examples/operator @askdba
+/examples/region_sharding @deepthi
+/java/ @harshit-gangal
+/go/cache @vmg
 /go/cmd/vtadmin @ajm188 @doeg
 /go/cmd/vtctldclient @ajm188 @doeg
 /go/mysql @harshit-gangal @systay
+/go/protoutil @ajm188
 /go/test/endtoend/onlineddl @shlomi-noach
 /go/test/endtoend/orchestrator @deepthi @shlomi-noach
 /go/test/endtoend/vtgate @harshit-gangal @systay
 /go/vt/discovery @deepthi
 /go/vt/mysqlctl @deepthi
-/go/vt/proto/vtadmin @ajm188 @doeg
 /go/vt/orchestrator @deepthi @shlomi-noach
+/go/vt/proto/vtadmin @ajm188 @doeg
 /go/vt/schema @shlomi-noach
-/go/vt/sqlparser @harshit-gangal @systay
-/go/vt/vtadmin @ajm188 @doeg
+/go/vt/servenv @deepthi
+/go/vt/sqlparser @harshit-gangal @systay @GuptaManan100
+/go/vt/srvtopo @rafael
+/go/vt/topo @deepthi @rafael
+/go/vt/vtadmin @ajm188 @doeg @rohit-nayak-ps
 /go/vt/vtctl @deepthi
 /go/vt/vtctl/vtctl.go @ajm188 @doeg
 /go/vt/vtctl/grpcvtctldclient @ajm188 @doeg
 /go/vt/vtctl/grpcvtctldserver @ajm188 @doeg
 /go/vt/vtctl/vtctldclient @ajm188 @doeg
+/go/vt/vtctld @ajm188 @doeg @rohit-nayak-ps @deepthi
 /go/vt/vtgate @harshit-gangal @systay
 /go/vt/vttablet/tabletmanager @deepthi @shlomi-noach
 /go/vt/vttablet/tabletmanager/vreplication @rohit-nayak-ps
 /go/vt/vttablet/tabletmanager/vstreamer @rohit-nayak-ps
 /go/vt/vttablet/tabletserver @harshit-gangal @systay @shlomi-noach
 /go/vt/wrangler @deepthi @rohit-nayak-ps
+/go/vt/workflow @rohit-nayak-ps
 /helm/ @derekperkins @dkhenry
 /proto/vtadmin.proto @ajm188 @doeg
 /proto/vtctldata.proto @ajm188 @doeg

.github/pull_request_template.md

@@ -1,3 +1,9 @@
+<!--
+  How to contribute: https://vitess.io/docs/contributing/
+  Please first make sure there is an open Issue to discuss the feature/fix suggested in this PR.
+  If this is a new feature, please mark the Issue as "RFC".
+ -->
+
 <!-- if this PR is Work in Progress please create it as a Draft Pull Request -->
 
 ## Description
@@ -22,4 +28,5 @@ Components that this PR will affect:
 - [ ]  Query Serving
 - [ ]  VReplication
 - [ ]  Cluster Management
-- [ ]  Build 
+- [ ]  Build/CI
+- [ ]  VTAdmin

.github/workflows/check_make_sizegen.yml

@@ -0,0 +1,35 @@
+name: check_make_sizegen
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Check Make Sizegen
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          go mod download
+
+      - name: Run make minimaltools
+        run: |
+          make minimaltools
+
+      - name: check_make_sizegen
+        run: |
+          tools/check_make_sizegen.sh
+

.github/workflows/check_make_visitor.yml

@@ -31,5 +31,5 @@ jobs:
 
     - name: check_make_visitor
       run: |
-        misc/git/hooks/visitorgen
+        misc/git/hooks/asthelpers
 

.github/workflows/check_runs_analysis.yaml

@@ -0,0 +1,59 @@
+on:
+  schedule:
+    - cron:  '*/10 * * * *'
+
+jobs:
+  analyze:
+    if: github.repository == 'vitessio/vitess'
+    name: analyze_check_runs
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+      with:
+        ref: metrics
+    - name: analyze check runs
+      run: |
+        FAILED_RUNS_JSON="/tmp/failed_runs.json"
+        FAILED_RUNS_CSV="/tmp/failed_runs.csv"
+        WORKFLOW_FAILURES_FILE=".metrics/workflow-failures"
+
+        # Get latest failed check runs
+        curl \
+          -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "Content-type: application/json" \
+          "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs?per_page=100&status=failure" \
+          > "$FAILED_RUNS_JSON"
+
+        # extract .created_at, .name, .html_url for each failed check run
+        cat "$FAILED_RUNS_JSON" |
+          jq -r '.workflow_runs[] | (.created_at + "," + .name + "," + .html_url)' > "$FAILED_RUNS_CSV"
+
+        # now combine back into $WORKFLOW_FAILURES_FILE,
+        # - skipping duplicates
+        # - limit file size to X last lines
+        # - encode with bas64
+        tmpfile="$(mktemp)"
+        cat $WORKFLOW_FAILURES_FILE $FAILED_RUNS_CSV | sort | uniq | tail -10000 | base64 -w 0 > $tmpfile
+        # sha of the file we're replacing:
+        sha="$(git hash-object ${WORKFLOW_FAILURES_FILE})"
+
+        json="
+          {
+            \"message\": \"automated merge of failed check runs\",
+            \"branch\": \"metrics\",
+            \"sha\": \"$sha\",
+            \"content\": \"$(cat $tmpfile)\"
+          }
+        "
+        json_tmpfile="$(mktemp)"
+        echo "$json" > $json_tmpfile
+
+        curl \
+          -X PUT \
+          -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "Content-type: application/json" \
+          "https://api.github.com/repos/${GITHUB_REPOSITORY}/contents/${WORKFLOW_FAILURES_FILE}" \
+          -d "@${json_tmpfile}"

.github/workflows/cluster_endtoend_11.yml

@@ -0,0 +1,50 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (11)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (11)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 11

.github/workflows/cluster_endtoend_12.yml

@@ -0,0 +1,50 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (12)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (12)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 12

.github/workflows/cluster_endtoend_13.yml

@@ -0,0 +1,50 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (13)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (13)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 13

.github/workflows/cluster_endtoend_14.yml

@@ -0,0 +1,50 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (14)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (14)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 14